Issue #18203: Replace malloc() with PyMem_RawMalloc() at Python initialization

* Replace malloc() with PyMem_RawMalloc()
* Replace PyMem_Malloc() with PyMem_RawMalloc() where the GIL is not held.
* _Py_char2wchar() now returns a buffer allocated by PyMem_RawMalloc(), instead
  of PyMem_Malloc()

Modules/getpath.c

@@ -343,7 +343,7 @@ search_for_prefix(wchar_t *argv0_path, wchar_t *home, wchar_t *_prefix)
         if (vpath != NULL) {
             wcscpy(prefix, argv0_path);
             joinpath(prefix, vpath);
-            PyMem_Free(vpath);
+            PyMem_RawFree(vpath);
             joinpath(prefix, L"Lib");
             joinpath(prefix, LANDMARK);
             if (ismodule(prefix))
@@ -554,8 +554,7 @@ calculate_path(void)
     }
     else
         progpath[0] = '\0';
-    if (path_buffer != NULL)
+    PyMem_RawFree(path_buffer);
-        PyMem_Free(path_buffer);
     if (progpath[0] != SEP && progpath[0] != '\0')
         absolutize(progpath);
     wcsncpy(argv0_path, progpath, MAXPATHLEN);
@@ -597,7 +596,7 @@ calculate_path(void)
             /* Use the location of the library as the progpath */
             wcsncpy(argv0_path, wbuf, MAXPATHLEN);
         }
-        PyMem_Free(wbuf);
+        PyMem_RawFree(wbuf);
     }
 #endif
 
@@ -808,11 +807,10 @@ calculate_path(void)
     else
         wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN);
 
-    PyMem_Free(_pythonpath);
+    PyMem_RawFree(_pythonpath);
-    PyMem_Free(_prefix);
+    PyMem_RawFree(_prefix);
-    PyMem_Free(_exec_prefix);
+    PyMem_RawFree(_exec_prefix);
-    if (rtpypath != NULL)
+    PyMem_RawFree(rtpypath);
-        PyMem_Free(rtpypath);
 }
 
 
@@ -822,7 +820,7 @@ Py_SetPath(const wchar_t *path)
 {
     if (module_search_path != NULL) {
         if (module_search_path_malloced)
-            PyMem_Free(module_search_path);
+            PyMem_RawFree(module_search_path);
         module_search_path = NULL;
         module_search_path_malloced = 0;
     }
@@ -831,7 +829,7 @@ Py_SetPath(const wchar_t *path)
         wchar_t *prog = Py_GetProgramName();
         wcsncpy(progpath, prog, MAXPATHLEN);
         exec_prefix[0] = prefix[0] = L'\0';
-        module_search_path = PyMem_Malloc((wcslen(path) + 1) * sizeof(wchar_t));
+        module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t));
         module_search_path_malloced = 1;
         if (module_search_path != NULL)
             wcscpy(module_search_path, path);

Modules/main.c

@@ -391,7 +391,7 @@ Py_Main(int argc, wchar_t **argv)
                command to interpret. */
 
             len = wcslen(_PyOS_optarg) + 1 + 1;
-            command = (wchar_t *)malloc(sizeof(wchar_t) * len);
+            command = (wchar_t *)PyMem_RawMalloc(sizeof(wchar_t) * len);
             if (command == NULL)
                 Py_FatalError(
                    "not enough memory to copy -c argument");
@@ -520,7 +520,7 @@ Py_Main(int argc, wchar_t **argv)
         *wp != L'\0') {
         wchar_t *buf, *warning;
 
-        buf = (wchar_t *)malloc((wcslen(wp) + 1) * sizeof(wchar_t));
+        buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t));
         if (buf == NULL)
             Py_FatalError(
                "not enough memory to copy PYTHONWARNINGS");
@@ -530,7 +530,7 @@ Py_Main(int argc, wchar_t **argv)
              warning = wcstok(NULL, L",")) {
             PySys_AddWarnOption(warning);
         }
-        free(buf);
+        PyMem_RawFree(buf);
     }
 #else
     if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
@@ -539,7 +539,7 @@ Py_Main(int argc, wchar_t **argv)
 
         /* settle for strtok here as there's no one standard
            C89 wcstok */
-        buf = (char *)malloc(strlen(p) + 1);
+        buf = (char *)PyMem_RawMalloc(strlen(p) + 1);
         if (buf == NULL)
             Py_FatalError(
                "not enough memory to copy PYTHONWARNINGS");
@@ -563,7 +563,7 @@ Py_Main(int argc, wchar_t **argv)
         }
         setlocale(LC_ALL, oldloc);
         free(oldloc);
-        free(buf);
+        PyMem_RawFree(buf);
     }
 #endif
 
@@ -633,7 +633,7 @@ Py_Main(int argc, wchar_t **argv)
         wchar_t* buffer;
         size_t len = strlen(p) + 1;
 
-        buffer = malloc(len * sizeof(wchar_t));
+        buffer = PyMem_RawMalloc(len * sizeof(wchar_t));
         if (buffer == NULL) {
             Py_FatalError(
                "not enough memory to copy PYTHONEXECUTABLE");
@@ -707,7 +707,7 @@ Py_Main(int argc, wchar_t **argv)
 
     if (command) {
         sts = run_command(command, &cf);
-        free(command);
+        PyMem_RawFree(command);
     } else if (module) {
         sts = (RunModule(module, 1) != 0);
     }

Modules/python.c

@@ -18,11 +18,19 @@ wmain(int argc, wchar_t **argv)
 int
 main(int argc, char **argv)
 {
-    wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
+    wchar_t **argv_copy;
     /* We need a second copies, as Python might modify the first one. */
-    wchar_t **argv_copy2 = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*(argc+1));
+    wchar_t **argv_copy2;
     int i, res;
     char *oldloc;
+
+    argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+    argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+    if (!argv_copy || !argv_copy2) {
+        fprintf(stderr, "out of memory\n");
+        return 1;
+    }
+
     /* 754 requires that FP exceptions run in "no stop" mode by default,
      * and until C vendors implement C99's ways to control FP exceptions,
      * Python requires non-stop mode.  Alas, some platforms enable FP
@@ -34,10 +42,7 @@ main(int argc, char **argv)
     m = fpgetmask();
     fpsetmask(m & ~FP_X_OFL);
 #endif
-    if (!argv_copy || !argv_copy2) {
+
-        fprintf(stderr, "out of memory\n");
-        return 1;
-    }
     oldloc = strdup(setlocale(LC_ALL, NULL));
     setlocale(LC_ALL, "");
     for (i = 0; i < argc; i++) {
@@ -57,10 +62,10 @@ main(int argc, char **argv)
     free(oldloc);
     res = Py_Main(argc, argv_copy);
     for (i = 0; i < argc; i++) {
-        PyMem_Free(argv_copy2[i]);
+        PyMem_RawFree(argv_copy2[i]);
     }
-    PyMem_Free(argv_copy);
+    PyMem_RawFree(argv_copy);
-    PyMem_Free(argv_copy2);
+    PyMem_RawFree(argv_copy2);
     return res;
 }
 #endif

Objects/unicodeobject.c

@@ -3316,7 +3316,7 @@ encode_error:
         wstr = _Py_char2wchar(errmsg, &errlen);
         if (wstr != NULL) {
             reason = PyUnicode_FromWideChar(wstr, errlen);
-            PyMem_Free(wstr);
+            PyMem_RawFree(wstr);
         } else
             errmsg = NULL;
     }
@@ -3535,7 +3535,7 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
         }
 
         unicode = PyUnicode_FromWideChar(wstr, wlen);
-        PyMem_Free(wstr);
+        PyMem_RawFree(wstr);
     }
     else {
         /* strict mode */
@@ -3583,7 +3583,7 @@ decode_error:
         wstr = _Py_char2wchar(errmsg, &errlen);
         if (wstr != NULL) {
             reason = PyUnicode_FromWideChar(wstr, errlen);
-            PyMem_Free(wstr);
+            PyMem_RawFree(wstr);
         } else
             errmsg = NULL;
     }

PC/getpathp.c

@@ -245,9 +245,9 @@ getpythonregpath(HKEY keyBase, int skipcore)
     /* Tried to use sysget("winver") but here is too early :-( */
     versionLen = strlen(PyWin_DLLVersionString);
     /* Space for all the chars, plus one \0 */
-    keyBuf = keyBufPtr = malloc(sizeof(keyPrefix) +
+    keyBuf = keyBufPtr = PyMem_RawMalloc(sizeof(keyPrefix) +
-                                sizeof(WCHAR)*(versionLen-1) +
+                                         sizeof(WCHAR)*(versionLen-1) +
-                                sizeof(keySuffix));
+                                         sizeof(keySuffix));
     if (keyBuf==NULL) goto done;
 
     memcpy(keyBufPtr, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR));
@@ -271,7 +271,7 @@ getpythonregpath(HKEY keyBase, int skipcore)
     /* Allocate a temp array of char buffers, so we only need to loop
        reading the registry once
     */
-    ppPaths = malloc( sizeof(WCHAR *) * numKeys );
+    ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys );
     if (ppPaths==NULL) goto done;
     memset(ppPaths, 0, sizeof(WCHAR *) * numKeys);
     /* Loop over all subkeys, allocating a temp sub-buffer. */
@@ -293,7 +293,7 @@ getpythonregpath(HKEY keyBase, int skipcore)
         /* Find the value of the buffer size, malloc, then read it */
         RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize);
         if (reqdSize) {
-            ppPaths[index] = malloc(reqdSize);
+            ppPaths[index] = PyMem_RawMalloc(reqdSize);
             if (ppPaths[index]) {
                 RegQueryValueExW(subKey, NULL, 0, NULL,
                                 (LPBYTE)ppPaths[index],
@@ -308,7 +308,7 @@ getpythonregpath(HKEY keyBase, int skipcore)
     if (dataSize == 0) goto done;
 
     /* original datasize from RegQueryInfo doesn't include the \0 */
-    dataBuf = malloc((dataSize+1) * sizeof(WCHAR));
+    dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR));
     if (dataBuf) {
         WCHAR *szCur = dataBuf;
         DWORD reqdSize = dataSize;
@@ -346,14 +346,13 @@ getpythonregpath(HKEY keyBase, int skipcore)
 done:
     /* Loop freeing my temp buffers */
     if (ppPaths) {
-        for(index=0;index<numKeys;index++)
+        for(index=0; index<numKeys; index++)
-            if (ppPaths[index]) free(ppPaths[index]);
+            PyMem_RawFree(ppPaths[index]);
-        free(ppPaths);
+        PyMem_RawFree(ppPaths);
     }
     if (newKey)
         RegCloseKey(newKey);
-    if (keyBuf)
+    PyMem_RawFree(keyBuf);
-        free(keyBuf);
     return retval;
 }
 #endif /* Py_ENABLE_SHARED */
@@ -616,7 +615,7 @@ calculate_path(void)
     if (envpath != NULL)
         bufsz += wcslen(envpath) + 1;
 
-    module_search_path = buf = malloc(bufsz*sizeof(wchar_t));
+    module_search_path = buf = PyMem_RawMalloc(bufsz*sizeof(wchar_t));
     if (buf == NULL) {
         /* We can't exit, so print a warning and limp along */
         fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n");
@@ -629,10 +628,8 @@ calculate_path(void)
             module_search_path = PYTHONPATH;
         }
 #ifdef MS_WINDOWS
-        if (machinepath)
+        PyMem_RawFree(machinepath);
-            free(machinepath);
+        PyMem_RawFree(userpath);
-        if (userpath)
-            free(userpath);
 #endif /* MS_WINDOWS */
         return;
     }
@@ -652,13 +649,13 @@ calculate_path(void)
         wcscpy(buf, userpath);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
-        free(userpath);
+        PyMem_RawFree(userpath);
     }
     if (machinepath) {
         wcscpy(buf, machinepath);
         buf = wcschr(buf, L'\0');
         *buf++ = DELIM;
-        free(machinepath);
+        PyMem_RawFree(machinepath);
     }
     if (pythonhome == NULL) {
         if (!skipdefault) {
@@ -745,7 +742,7 @@ void
 Py_SetPath(const wchar_t *path)
 {
     if (module_search_path != NULL) {
-        free(module_search_path);
+        PyMem_RawFree(module_search_path);
         module_search_path = NULL;
     }
     if (path != NULL) {
@@ -753,10 +750,10 @@ Py_SetPath(const wchar_t *path)
         wchar_t *prog = Py_GetProgramName();
         wcsncpy(progpath, prog, MAXPATHLEN);
         prefix[0] = L'\0';
-        module_search_path = malloc((wcslen(path) + 1) * sizeof(wchar_t));
+        module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t));
         if (module_search_path != NULL)
             wcscpy(module_search_path, path);
-	}
+    }
 }
 
 wchar_t *

Python/fileutils.c

@@ -229,7 +229,7 @@ decode_ascii_surrogateescape(const char *arg, size_t *size)
    Use _Py_wchar2char() to encode the character string back to a byte string.
 
    Return a pointer to a newly allocated wide character string (use
-   PyMem_Free() to free the memory) and write the number of written wide
+   PyMem_RawFree() to free the memory) and write the number of written wide
    characters excluding the null character into *size if size is not NULL, or
    NULL on error (decoding or memory allocation error). If size is not NULL,
    *size is set to (size_t)-1 on memory error and (size_t)-2 on decoding
@@ -283,7 +283,7 @@ _Py_char2wchar(const char* arg, size_t *size)
     argsize = mbstowcs(NULL, arg, 0);
 #endif
     if (argsize != (size_t)-1) {
-        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        res = (wchar_t *)PyMem_RawMalloc((argsize+1)*sizeof(wchar_t));
         if (!res)
             goto oom;
         count = mbstowcs(res, arg, argsize+1);
@@ -300,7 +300,7 @@ _Py_char2wchar(const char* arg, size_t *size)
                 return res;
             }
         }
-        PyMem_Free(res);
+        PyMem_RawFree(res);
     }
     /* Conversion failed. Fall back to escaping with surrogateescape. */
 #ifdef HAVE_MBRTOWC
@@ -309,7 +309,7 @@ _Py_char2wchar(const char* arg, size_t *size)
     /* Overallocate; as multi-byte characters are in the argument, the
        actual output could use less memory. */
     argsize = strlen(arg) + 1;
-    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+    res = (wchar_t*)PyMem_RawMalloc(argsize*sizeof(wchar_t));
     if (!res)
         goto oom;
     in = (unsigned char*)arg;
@@ -325,7 +325,7 @@ _Py_char2wchar(const char* arg, size_t *size)
                since we provide everything that we have -
                unless there is a bug in the C library, or I
                misunderstood how mbrtowc works. */
-            PyMem_Free(res);
+            PyMem_RawFree(res);
             if (size != NULL)
                 *size = (size_t)-2;
             return NULL;
@@ -648,12 +648,12 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz)
         return -1;
     }
     if (bufsiz <= r1) {
-        PyMem_Free(wbuf);
+        PyMem_RawFree(wbuf);
         errno = EINVAL;
         return -1;
     }
     wcsncpy(buf, wbuf, bufsiz);
-    PyMem_Free(wbuf);
+    PyMem_RawFree(wbuf);
     return (int)r1;
 }
 #endif
@@ -689,12 +689,12 @@ _Py_wrealpath(const wchar_t *path,
         return NULL;
     }
     if (resolved_path_size <= r) {
-        PyMem_Free(wresolved_path);
+        PyMem_RawFree(wresolved_path);
         errno = EINVAL;
         return NULL;
     }
     wcsncpy(resolved_path, wresolved_path, resolved_path_size);
-    PyMem_Free(wresolved_path);
+    PyMem_RawFree(wresolved_path);
     return resolved_path;
 }
 #endif
@@ -720,11 +720,11 @@ _Py_wgetcwd(wchar_t *buf, size_t size)
     if (wname == NULL)
         return NULL;
     if (size <= len) {
-        PyMem_Free(wname);
+        PyMem_RawFree(wname);
         return NULL;
     }
     wcsncpy(buf, wname, size);
-    PyMem_Free(wname);
+    PyMem_RawFree(wname);
     return buf;
 #endif
 }

Python/pystate.c

@@ -6,11 +6,11 @@
 /* --------------------------------------------------------------------------
 CAUTION
 
-Always use malloc() and free() directly in this file.  A number of these
+Always use PyMem_RawMalloc() and PyMem_RawFree() directly in this file.  A
-functions are advertised as safe to call when the GIL isn't held, and in
+number of these functions are advertised as safe to call when the GIL isn't
-a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's debugging
+held, and in a debug build Python redirects (e.g.) PyMem_NEW (etc) to Python's
-obmalloc functions.  Those aren't thread-safe (they rely on the GIL to avoid
+debugging obmalloc functions.  Those aren't thread-safe (they rely on the GIL
-the expense of doing their own locking).
+to avoid the expense of doing their own locking).
 -------------------------------------------------------------------------- */
 
 #ifdef HAVE_DLOPEN
@@ -60,7 +60,7 @@ PyInterpreterState *
 PyInterpreterState_New(void)
 {
     PyInterpreterState *interp = (PyInterpreterState *)
-                                 malloc(sizeof(PyInterpreterState));
+                                 PyMem_RawMalloc(sizeof(PyInterpreterState));
 
     if (interp != NULL) {
         HEAD_INIT();
@@ -148,7 +148,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
         Py_FatalError("PyInterpreterState_Delete: remaining threads");
     *p = interp->next;
     HEAD_UNLOCK();
-    free(interp);
+    PyMem_RawFree(interp);
 #ifdef WITH_THREAD
     if (interp_head == NULL && head_mutex != NULL) {
         PyThread_free_lock(head_mutex);
@@ -168,7 +168,7 @@ threadstate_getframe(PyThreadState *self)
 static PyThreadState *
 new_threadstate(PyInterpreterState *interp, int init)
 {
-    PyThreadState *tstate = (PyThreadState *)malloc(sizeof(PyThreadState));
+    PyThreadState *tstate = (PyThreadState *)PyMem_RawMalloc(sizeof(PyThreadState));
 
     if (_PyThreadState_GetFrame == NULL)
         _PyThreadState_GetFrame = threadstate_getframe;
@@ -365,7 +365,7 @@ tstate_delete_common(PyThreadState *tstate)
     if (tstate->next)
         tstate->next->prev = tstate->prev;
     HEAD_UNLOCK();
-    free(tstate);
+    PyMem_RawFree(tstate);
 }
 
 
@@ -432,7 +432,7 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate)
     for (p = garbage; p; p = next) {
         next = p->next;
         PyThreadState_Clear(p);
-        free(p);
+        PyMem_RawFree(p);
     }
 }
 

Python/thread.c

@@ -231,7 +231,7 @@ find_key(int key, void *value)
         assert(p == NULL);
         goto Done;
     }
-    p = (struct key *)malloc(sizeof(struct key));
+    p = (struct key *)PyMem_RawMalloc(sizeof(struct key));
     if (p != NULL) {
         p->id = id;
         p->key = key;
@@ -270,7 +270,7 @@ PyThread_delete_key(int key)
     while ((p = *q) != NULL) {
         if (p->key == key) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
         }
         else
@@ -324,7 +324,7 @@ PyThread_delete_key_value(int key)
     while ((p = *q) != NULL) {
         if (p->key == key && p->id == id) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
             break;
         }
@@ -357,7 +357,7 @@ PyThread_ReInitTLS(void)
     while ((p = *q) != NULL) {
         if (p->id != id) {
             *q = p->next;
-            free((void *)p);
+            PyMem_RawFree((void *)p);
             /* NB This does *not* free p->value! */
         }
         else