Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),

PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int) are now
using ``calloc()`` instead of ``malloc()`` for large objects which is faster
and use less memory (until the bytearray buffer is filled with data).

Doc/c-api/memory.rst

@@ -92,8 +92,8 @@ functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
 need to be held.
 
 The default raw memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
 
 .. versionadded:: 3.4
 
@@ -106,6 +106,17 @@ requesting zero bytes.
    been initialized in any way.
 
 
+.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize)
+
+   Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+   a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+   request fails. The memory is initialized to zeros. Requesting zero elements
+   or elements of size zero bytes returns a distinct non-*NULL* pointer if
+   possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead.
+
+   .. versionadded:: 3.5
+
+
 .. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
 
    Resizes the memory block pointed to by *p* to *n* bytes. The contents will
@@ -136,8 +147,8 @@ behavior when requesting zero bytes, are available for allocating and releasing
 memory from the Python heap.
 
 The default memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
 
 .. warning::
 
@@ -152,6 +163,17 @@ requesting zero bytes.
    been called instead. The memory will not have been initialized in any way.
 
 
+.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize)
+
+   Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+   a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+   request fails. The memory is initialized to zeros. Requesting zero elements
+   or elements of size zero bytes returns a distinct non-*NULL* pointer if
+   possible, as if ``PyMem_Calloc(1, 1)`` had been called instead.
+
+   .. versionadded:: 3.5
+
+
 .. c:function:: void* PyMem_Realloc(void *p, size_t n)
 
    Resizes the memory block pointed to by *p* to *n* bytes. The contents will be
@@ -222,11 +244,17 @@ Customize Memory Allocators
    +----------------------------------------------------------+---------------------------------------+
    | ``void* malloc(void *ctx, size_t size)``                 | allocate a memory block               |
    +----------------------------------------------------------+---------------------------------------+
+   | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized   |
+   |                                                          | with zeros                            |
+   +----------------------------------------------------------+---------------------------------------+
    | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block     |
    +----------------------------------------------------------+---------------------------------------+
    | ``void free(void *ctx, void *ptr)``                      | free a memory block                   |
    +----------------------------------------------------------+---------------------------------------+
 
+   .. versionchanged:: 3.5
+      Add a new field ``calloc``.
+
 .. c:type:: PyMemAllocatorDomain
 
    Enum used to identify an allocator domain. Domains:

Doc/whatsnew/3.5.rst

@@ -164,7 +164,10 @@ Optimizations
 
 Major performance enhancements have been added:
 
-* None yet.
+* Construction of ``bytes(int)`` and ``bytearray(int)`` (filled by zero bytes)
+  is faster and use less memory (until the bytearray buffer is filled with
+  data) for large objects. ``calloc()`` is used instead of ``malloc()`` to
+  allocate memory for these objects.
 
 
 Build and C API Changes
@@ -172,7 +175,12 @@ Build and C API Changes
 
 Changes to Python's build process and to the C API include:
 
-* None yet.
+* New ``calloc`` functions:
+
+  * :c:func:`PyMem_RawCalloc`
+  * :c:func:`PyMem_Calloc`
+  * :c:func:`PyObject_Calloc`
+  * :c:func:`_PyObject_GC_Calloc`
 
 
 Deprecated
@@ -209,6 +217,9 @@ Porting to Python 3.5
 This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
+Changes in the Python API
+-------------------------
+
 * Before Python 3.5, a :class:`datetime.time` object was considered to be false
   if it represented midnight in UTC.  This behavior was considered obscure and
   error-prone and has been removed in Python 3.5.  See :issue:`13936` for full
@@ -217,3 +228,8 @@ that may require changes to your code.
 * :meth:`ssl.SSLSocket.send()` now raises either :exc:`ssl.SSLWantReadError`
   or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation
   would block. Previously, it would return 0.  See :issue:`20951`.
+
+Changes in the C API
+--------------------
+
+* The :c:type:`PyMemAllocator` structure has a new ``calloc`` field.

Include/objimpl.h

@@ -95,6 +95,7 @@ PyObject_{New, NewVar, Del}.
    the raw memory.
 */
 PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Calloc(size_t nelem, size_t elsize);
 PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyObject_Free(void *ptr);
 
@@ -321,7 +322,8 @@ extern PyGC_Head *_PyGC_generation0;
         (!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj)))
 #endif /* Py_LIMITED_API */
 
-PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size);
 PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
 PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
 PyAPI_FUNC(void) PyObject_GC_Track(void *);

Include/pymem.h

@@ -13,6 +13,7 @@ extern "C" {
 
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
 PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
 #endif
@@ -57,6 +58,7 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
 */
 
 PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
 PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
@@ -132,6 +134,9 @@ typedef struct {
     /* allocate a memory block */
     void* (*malloc) (void *ctx, size_t size);
 
+    /* allocate a memory block initialized by zeros */
+    void* (*calloc) (void *ctx, size_t nelem, size_t elsize);
+
     /* allocate or resize a memory block */
     void* (*realloc) (void *ctx, void *ptr, size_t new_size);
 

Misc/NEWS

@@ -10,6 +10,12 @@ Release date: TBA
 Core and Builtins
 -----------------
 
+- Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
+  PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int)
+  are now using ``calloc()`` instead of ``malloc()`` for large objects which
+  is faster and use less memory (until the bytearray buffer is filled with
+  data).
+
 - Issue #21377: PyBytes_Concat() now tries to concatenate in-place when the
   first argument has a reference count of 1.  Patch by Nikolaus Rath.
 

Modules/_testcapimodule.c

@@ -2710,6 +2710,20 @@ test_pymem_alloc0(PyObject *self)
 {
     void *ptr;
 
+    ptr = PyMem_RawMalloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
+    ptr = PyMem_RawCalloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawCalloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
     ptr = PyMem_Malloc(0);
     if (ptr == NULL) {
         PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
@@ -2717,6 +2731,13 @@ test_pymem_alloc0(PyObject *self)
     }
     PyMem_Free(ptr);
 
+    ptr = PyMem_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
     ptr = PyObject_Malloc(0);
     if (ptr == NULL) {
         PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
@@ -2724,6 +2745,13 @@ test_pymem_alloc0(PyObject *self)
     }
     PyObject_Free(ptr);
 
+    ptr = PyObject_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
     Py_RETURN_NONE;
 }
 
@@ -2731,6 +2759,8 @@ typedef struct {
     PyMemAllocator alloc;
 
     size_t malloc_size;
+    size_t calloc_nelem;
+    size_t calloc_elsize;
     void *realloc_ptr;
     size_t realloc_new_size;
     void *free_ptr;
@@ -2743,6 +2773,14 @@ static void* hook_malloc (void* ctx, size_t size)
     return hook->alloc.malloc(hook->alloc.ctx, size);
 }
 
+static void* hook_calloc (void* ctx, size_t nelem, size_t elsize)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->calloc_nelem = nelem;
+    hook->calloc_elsize = elsize;
+    return hook->alloc.calloc(hook->alloc.ctx, nelem, elsize);
+}
+
 static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
 {
     alloc_hook_t *hook = (alloc_hook_t *)ctx;
@@ -2765,16 +2803,14 @@ test_setallocators(PyMemAllocatorDomain domain)
     const char *error_msg;
     alloc_hook_t hook;
     PyMemAllocator alloc;
-    size_t size, size2;
+    size_t size, size2, nelem, elsize;
     void *ptr, *ptr2;
 
-    hook.malloc_size = 0;
-    hook.realloc_ptr = NULL;
-    hook.realloc_new_size = 0;
-    hook.free_ptr = NULL;
+    memset(&hook, 0, sizeof(hook));
 
     alloc.ctx = &hook;
     alloc.malloc = &hook_malloc;
+    alloc.calloc = &hook_calloc;
     alloc.realloc = &hook_realloc;
     alloc.free = &hook_free;
     PyMem_GetAllocator(domain, &hook.alloc);
@@ -2831,6 +2867,33 @@ test_setallocators(PyMemAllocatorDomain domain)
         goto fail;
     }
 
+    nelem = 2;
+    elsize = 5;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawCalloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Calloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Calloc(nelem, elsize); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "calloc failed";
+        goto fail;
+    }
+
+    if (hook.calloc_nelem != nelem || hook.calloc_elsize != elsize) {
+        error_msg = "calloc invalid nelem or elsize";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr); break;
+    }
+
     Py_INCREF(Py_None);
     res = Py_None;
     goto finally;

Modules/_tracemalloc.c

@@ -476,17 +476,22 @@ tracemalloc_remove_trace(void *ptr)
 }
 
 static void*
-tracemalloc_malloc(void *ctx, size_t size)
+tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
 {
     PyMemAllocator *alloc = (PyMemAllocator *)ctx;
     void *ptr;
 
-    ptr = alloc->malloc(alloc->ctx, size);
+    assert(nelem <= PY_SIZE_MAX / elsize);
+
+    if (use_calloc)
+        ptr = alloc->calloc(alloc->ctx, nelem, elsize);
+    else
+        ptr = alloc->malloc(alloc->ctx, nelem * elsize);
     if (ptr == NULL)
         return NULL;
 
     TABLES_LOCK();
-    if (tracemalloc_add_trace(ptr, size) < 0) {
+    if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) {
         /* Failed to allocate a trace for the new memory block */
         TABLES_UNLOCK();
         alloc->free(alloc->ctx, ptr);
@@ -560,13 +565,16 @@ tracemalloc_free(void *ctx, void *ptr)
 }
 
 static void*
-tracemalloc_malloc_gil(void *ctx, size_t size)
+tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize)
 {
     void *ptr;
 
     if (get_reentrant()) {
         PyMemAllocator *alloc = (PyMemAllocator *)ctx;
-        return alloc->malloc(alloc->ctx, size);
+        if (use_calloc)
+            return alloc->calloc(alloc->ctx, nelem, elsize);
+        else
+            return alloc->malloc(alloc->ctx, nelem * elsize);
     }
 
     /* Ignore reentrant call. PyObjet_Malloc() calls PyMem_Malloc() for
@@ -574,12 +582,24 @@ tracemalloc_malloc_gil(void *ctx, size_t size)
        allocation twice. */
     set_reentrant(1);
 
-    ptr = tracemalloc_malloc(ctx, size);
+    ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
 
     set_reentrant(0);
     return ptr;
 }
 
+static void*
+tracemalloc_malloc_gil(void *ctx, size_t size)
+{
+    return tracemalloc_alloc_gil(0, ctx, 1, size);
+}
+
+static void*
+tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize)
+{
+    return tracemalloc_alloc_gil(1, ctx, nelem, elsize);
+}
+
 static void*
 tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size)
 {
@@ -614,7 +634,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size)
 
 #ifdef TRACE_RAW_MALLOC
 static void*
-tracemalloc_raw_malloc(void *ctx, size_t size)
+tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
 {
 #ifdef WITH_THREAD
     PyGILState_STATE gil_state;
@@ -623,7 +643,10 @@ tracemalloc_raw_malloc(void *ctx, size_t size)
 
     if (get_reentrant()) {
         PyMemAllocator *alloc = (PyMemAllocator *)ctx;
-        return alloc->malloc(alloc->ctx, size);
+        if (use_calloc)
+            return alloc->calloc(alloc->ctx, nelem, elsize);
+        else
+            return alloc->malloc(alloc->ctx, nelem * elsize);
     }
 
     /* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc()
@@ -633,16 +656,28 @@ tracemalloc_raw_malloc(void *ctx, size_t size)
 
 #ifdef WITH_THREAD
     gil_state = PyGILState_Ensure();
-    ptr = tracemalloc_malloc(ctx, size);
+    ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
     PyGILState_Release(gil_state);
 #else
-    ptr = tracemalloc_malloc(ctx, size);
+    ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
 #endif
 
     set_reentrant(0);
     return ptr;
 }
 
+static void*
+tracemalloc_raw_malloc(void *ctx, size_t size)
+{
+    return tracemalloc_raw_alloc(0, ctx, 1, size);
+}
+
+static void*
+tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize)
+{
+    return tracemalloc_raw_alloc(1, ctx, nelem, elsize);
+}
+
 static void*
 tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size)
 {
@@ -856,6 +891,7 @@ tracemalloc_start(int max_nframe)
 
 #ifdef TRACE_RAW_MALLOC
     alloc.malloc = tracemalloc_raw_malloc;
+    alloc.calloc = tracemalloc_raw_calloc;
     alloc.realloc = tracemalloc_raw_realloc;
     alloc.free = tracemalloc_free;
 
@@ -865,6 +901,7 @@ tracemalloc_start(int max_nframe)
 #endif
 
     alloc.malloc = tracemalloc_malloc_gil;
+    alloc.calloc = tracemalloc_calloc_gil;
     alloc.realloc = tracemalloc_realloc_gil;
     alloc.free = tracemalloc_free;
 

Modules/gcmodule.c

@@ -1703,15 +1703,19 @@ PyObject_GC_UnTrack(void *op)
         _PyObject_GC_UNTRACK(op);
 }
 
-PyObject *
-_PyObject_GC_Malloc(size_t basicsize)
+static PyObject *
+_PyObject_GC_Alloc(int use_calloc, size_t basicsize)
 {
     PyObject *op;
     PyGC_Head *g;
+    size_t size;
     if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
         return PyErr_NoMemory();
-    g = (PyGC_Head *)PyObject_MALLOC(
-        sizeof(PyGC_Head) + basicsize);
+    size = sizeof(PyGC_Head) + basicsize;
+    if (use_calloc)
+        g = (PyGC_Head *)PyObject_Calloc(1, size);
+    else
+        g = (PyGC_Head *)PyObject_Malloc(size);
     if (g == NULL)
         return PyErr_NoMemory();
     g->gc.gc_refs = 0;
@@ -1730,6 +1734,18 @@ _PyObject_GC_Malloc(size_t basicsize)
     return op;
 }
 
+PyObject *
+_PyObject_GC_Malloc(size_t basicsize)
+{
+    return _PyObject_GC_Alloc(0, basicsize);
+}
+
+PyObject *
+_PyObject_GC_Calloc(size_t basicsize)
+{
+    return _PyObject_GC_Alloc(1, basicsize);
+}
+
 PyObject *
 _PyObject_GC_New(PyTypeObject *tp)
 {

Objects/bytearrayobject.c

@@ -813,9 +813,21 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds)
     }
     else {
         if (count > 0) {
-            if (PyByteArray_Resize((PyObject *)self, count))
+            void *sval;
+            Py_ssize_t alloc;
+
+            assert (Py_SIZE(self) == 0);
+
+            alloc = count + 1;
+            sval = PyObject_Calloc(1, alloc);
+            if (sval == NULL)
                 return -1;
-            memset(PyByteArray_AS_STRING(self), 0, count);
+
+            PyObject_Free(self->ob_bytes);
+
+            self->ob_bytes = self->ob_start = sval;
+            Py_SIZE(self) = count;
+            self->ob_alloc = alloc;
         }
         return 0;
     }

Objects/bytesobject.c

@@ -71,15 +71,11 @@ static PyBytesObject *nullstring;
    PyBytes_FromStringAndSize()) or the length of the string in the `str'
    parameter (for PyBytes_FromString()).
 */
-PyObject *
-PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
+static PyObject *
+_PyBytes_FromSize(Py_ssize_t size, int use_calloc)
 {
     PyBytesObject *op;
-    if (size < 0) {
-        PyErr_SetString(PyExc_SystemError,
-            "Negative size passed to PyBytes_FromStringAndSize");
-        return NULL;
-    }
+    assert(size >= 0);
     if (size == 0 && (op = nullstring) != NULL) {
 #ifdef COUNT_ALLOCS
         null_strings++;
@@ -87,15 +83,6 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
         Py_INCREF(op);
         return (PyObject *)op;
     }
-    if (size == 1 && str != NULL &&
-        (op = characters[*str & UCHAR_MAX]) != NULL)
-    {
-#ifdef COUNT_ALLOCS
-        one_strings++;
-#endif
-        Py_INCREF(op);
-        return (PyObject *)op;
-    }
 
     if (size > PY_SSIZE_T_MAX - PyBytesObject_SIZE) {
         PyErr_SetString(PyExc_OverflowError,
@@ -104,19 +91,52 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
     }
 
     /* Inline PyObject_NewVar */
-    op = (PyBytesObject *)PyObject_MALLOC(PyBytesObject_SIZE + size);
+    if (use_calloc)
+        op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size);
+    else
+        op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size);
     if (op == NULL)
         return PyErr_NoMemory();
     (void)PyObject_INIT_VAR(op, &PyBytes_Type, size);
     op->ob_shash = -1;
-    if (str != NULL)
-        Py_MEMCPY(op->ob_sval, str, size);
+    if (!use_calloc)
         op->ob_sval[size] = '\0';
-    /* share short strings */
+    /* empty byte string singleton */
     if (size == 0) {
         nullstring = op;
         Py_INCREF(op);
-    } else if (size == 1 && str != NULL) {
+    }
+    return (PyObject *) op;
+}
+
+PyObject *
+PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
+{
+    PyBytesObject *op;
+    if (size < 0) {
+        PyErr_SetString(PyExc_SystemError,
+            "Negative size passed to PyBytes_FromStringAndSize");
+        return NULL;
+    }
+    if (size == 1 && str != NULL &&
+        (op = characters[*str & UCHAR_MAX]) != NULL)
+    {
+#ifdef COUNT_ALLOCS
+        one_strings++;
+#endif
+        Py_INCREF(op);
+        return (PyObject *)op;
+    }
+
+    op = (PyBytesObject *)_PyBytes_FromSize(size, 0);
+    if (op == NULL)
+        return NULL;
+    if (str == NULL)
+        return (PyObject *) op;
+
+    Py_MEMCPY(op->ob_sval, str, size);
+    /* share short strings */
+    if (size == 1) {
         characters[*str & UCHAR_MAX] = op;
         Py_INCREF(op);
     }
@@ -2482,7 +2502,7 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
                             "argument");
             return NULL;
         }
-        return PyBytes_FromString("");
+        return PyBytes_FromStringAndSize(NULL, 0);
     }
 
     if (PyUnicode_Check(x)) {
@@ -2532,11 +2552,9 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
     else {
-        new = PyBytes_FromStringAndSize(NULL, size);
+        new = _PyBytes_FromSize(size, 1);
         if (new == NULL)
             return NULL;
-        if (size > 0)
-            memset(((PyBytesObject*)new)->ob_sval, 0, size);
         return new;
     }
 

Objects/obmalloc.c

@@ -5,6 +5,7 @@
 #ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
 /* Forward declaration */
 static void* _PyMem_DebugMalloc(void *ctx, size_t size);
+static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);
 static void _PyMem_DebugFree(void *ctx, void *p);
 static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);
 
@@ -43,6 +44,7 @@ static void _PyMem_DebugCheckAddress(char api_id, const void *p);
 
 /* Forward declaration */
 static void* _PyObject_Malloc(void *ctx, size_t size);
+static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);
 static void _PyObject_Free(void *ctx, void *p);
 static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
 #endif
@@ -51,7 +53,7 @@ static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
 static void *
 _PyMem_RawMalloc(void *ctx, size_t size)
 {
-    /* PyMem_Malloc(0) means malloc(1). Some systems would return NULL
+    /* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL
        for malloc(0), which would be treated as an error. Some platforms would
        return a pointer with no memory behind it, which would break pymalloc.
        To solve these problems, allocate an extra byte. */
@@ -60,6 +62,20 @@ _PyMem_RawMalloc(void *ctx, size_t size)
     return malloc(size);
 }
 
+static void *
+_PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize)
+{
+    /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL
+       for calloc(0, 0), which would be treated as an error. Some platforms
+       would return a pointer with no memory behind it, which would break
+       pymalloc.  To solve these problems, allocate an extra byte. */
+    if (nelem == 0 || elsize == 0) {
+        nelem = 1;
+        elsize = 1;
+    }
+    return calloc(nelem, elsize);
+}
+
 static void *
 _PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
 {
@@ -123,9 +139,9 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size)
 #endif
 
 
-#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree
+#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree
 #ifdef WITH_PYMALLOC
-#  define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free
+#  define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free
 #else
 #  define PYOBJ_FUNCS PYRAW_FUNCS
 #endif
@@ -147,7 +163,7 @@ static struct {
     {'o', {NULL, PYOBJ_FUNCS}}
     };
 
-#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
+#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
 #endif
 
 static PyMemAllocator _PyMem_Raw = {
@@ -196,6 +212,7 @@ PyMem_SetupDebugHooks(void)
     PyMemAllocator alloc;
 
     alloc.malloc = _PyMem_DebugMalloc;
+    alloc.calloc = _PyMem_DebugCalloc;
     alloc.realloc = _PyMem_DebugRealloc;
     alloc.free = _PyMem_DebugFree;
 
@@ -228,9 +245,10 @@ PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
     case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;
     case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;
     default:
-        /* unknown domain */
+        /* unknown domain: set all attributes to NULL */
         allocator->ctx = NULL;
         allocator->malloc = NULL;
+        allocator->calloc = NULL;
         allocator->realloc = NULL;
         allocator->free = NULL;
     }
@@ -272,10 +290,18 @@ PyMem_RawMalloc(size_t size)
      */
     if (size > (size_t)PY_SSIZE_T_MAX)
         return NULL;
-
     return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
 }
 
+void *
+PyMem_RawCalloc(size_t nelem, size_t elsize)
+{
+    /* see PyMem_RawMalloc() */
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);
+}
+
 void*
 PyMem_RawRealloc(void *ptr, size_t new_size)
 {
@@ -299,6 +325,15 @@ PyMem_Malloc(size_t size)
     return _PyMem.malloc(_PyMem.ctx, size);
 }
 
+void *
+PyMem_Calloc(size_t nelem, size_t elsize)
+{
+    /* see PyMem_RawMalloc() */
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    return _PyMem.calloc(_PyMem.ctx, nelem, elsize);
+}
+
 void *
 PyMem_Realloc(void *ptr, size_t new_size)
 {
@@ -351,6 +386,15 @@ PyObject_Malloc(size_t size)
     return _PyObject.malloc(_PyObject.ctx, size);
 }
 
+void *
+PyObject_Calloc(size_t nelem, size_t elsize)
+{
+    /* see PyMem_RawMalloc() */
+    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+        return NULL;
+    return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
+}
+
 void *
 PyObject_Realloc(void *ptr, size_t new_size)
 {
@@ -1122,8 +1166,9 @@ int Py_ADDRESS_IN_RANGE(void *P, poolp pool) Py_NO_INLINE;
  */
 
 static void *
-_PyObject_Malloc(void *ctx, size_t nbytes)
+_PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
 {
+    size_t nbytes;
     block *bp;
     poolp pool;
     poolp next;
@@ -1138,9 +1183,12 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
         goto redirect;
 #endif
 
-    /*
-     * This implicitly redirects malloc(0).
-     */
+    if (nelem == 0 || elsize == 0)
+        goto redirect;
+
+    assert(nelem <= PY_SSIZE_T_MAX / elsize);
+    nbytes = nelem * elsize;
+
     if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
         LOCK();
         /*
@@ -1158,6 +1206,8 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
             assert(bp != NULL);
             if ((pool->freeblock = *(block **)bp) != NULL) {
                 UNLOCK();
+                if (use_calloc)
+                    memset(bp, 0, nbytes);
                 return (void *)bp;
             }
             /*
@@ -1170,6 +1220,8 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
                 pool->nextoffset += INDEX2SIZE(size);
                 *(block **)(pool->freeblock) = NULL;
                 UNLOCK();
+                if (use_calloc)
+                    memset(bp, 0, nbytes);
                 return (void *)bp;
             }
             /* Pool is full, unlink from used pools. */
@@ -1178,6 +1230,8 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
             next->prevpool = pool;
             pool->nextpool = next;
             UNLOCK();
+            if (use_calloc)
+                memset(bp, 0, nbytes);
             return (void *)bp;
         }
 
@@ -1257,6 +1311,8 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
                 assert(bp != NULL);
                 pool->freeblock = *(block **)bp;
                 UNLOCK();
+                if (use_calloc)
+                    memset(bp, 0, nbytes);
                 return (void *)bp;
             }
             /*
@@ -1272,6 +1328,8 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
             pool->freeblock = bp + size;
             *(block **)(pool->freeblock) = NULL;
             UNLOCK();
+            if (use_calloc)
+                memset(bp, 0, nbytes);
             return (void *)bp;
         }
 
@@ -1311,13 +1369,29 @@ redirect:
      * has been reached.
      */
     {
-        void *result = PyMem_RawMalloc(nbytes);
+        void *result;
+        if (use_calloc)
+            result = PyMem_RawCalloc(nelem, elsize);
+        else
+            result = PyMem_RawMalloc(nbytes);
         if (!result)
             _Py_AllocatedBlocks--;
         return result;
     }
 }
 
+static void *
+_PyObject_Malloc(void *ctx, size_t nbytes)
+{
+    return _PyObject_Alloc(0, ctx, 1, nbytes);
+}
+
+static void *
+_PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
+{
+    return _PyObject_Alloc(1, ctx, nelem, elsize);
+}
+
 /* free */
 
 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
@@ -1561,7 +1635,7 @@ _PyObject_Realloc(void *ctx, void *p, size_t nbytes)
 #endif
 
     if (p == NULL)
-        return _PyObject_Malloc(ctx, nbytes);
+        return _PyObject_Alloc(0, ctx, 1, nbytes);
 
 #ifdef WITH_VALGRIND
     /* Treat running_on_valgrind == -1 the same as 0 */
@@ -1589,7 +1663,7 @@ _PyObject_Realloc(void *ctx, void *p, size_t nbytes)
             }
             size = nbytes;
         }
-        bp = _PyObject_Malloc(ctx, nbytes);
+        bp = _PyObject_Alloc(0, ctx, 1, nbytes);
         if (bp != NULL) {
             memcpy(bp, p, size);
             _PyObject_Free(ctx, p);
@@ -1745,7 +1819,7 @@ p[2*S+n+S: 2*S+n+2*S]
 */
 
 static void *
-_PyMem_DebugMalloc(void *ctx, size_t nbytes)
+_PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes)
 {
     debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
     uchar *p;           /* base address of malloc'ed block */
@@ -1758,6 +1832,9 @@ _PyMem_DebugMalloc(void *ctx, size_t nbytes)
         /* overflow:  can't represent total as a size_t */
         return NULL;
 
+    if (use_calloc)
+        p = (uchar *)api->alloc.calloc(api->alloc.ctx, 1, total);
+    else
         p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
     if (p == NULL)
         return NULL;
@@ -1767,7 +1844,7 @@ _PyMem_DebugMalloc(void *ctx, size_t nbytes)
     p[SST] = (uchar)api->api_id;
     memset(p + SST + 1, FORBIDDENBYTE, SST-1);
 
-    if (nbytes > 0)
+    if (nbytes > 0 && !use_calloc)
         memset(p + 2*SST, CLEANBYTE, nbytes);
 
     /* at tail, write pad (SST bytes) and serialno (SST bytes) */
@@ -1778,6 +1855,21 @@ _PyMem_DebugMalloc(void *ctx, size_t nbytes)
     return p + 2*SST;
 }
 
+static void *
+_PyMem_DebugMalloc(void *ctx, size_t nbytes)
+{
+    return _PyMem_DebugAlloc(0, ctx, nbytes);
+}
+
+static void *
+_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize)
+{
+    size_t nbytes;
+    assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize);
+    nbytes = nelem * elsize;
+    return _PyMem_DebugAlloc(1, ctx, nbytes);
+}
+
 /* The debug free first checks the 2*SST bytes on each end for sanity (in
    particular, that the FORBIDDENBYTEs with the api ID are still intact).
    Then fills the original bytes with DEADBYTE.
@@ -1811,7 +1903,7 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
     int i;
 
     if (p == NULL)
-        return _PyMem_DebugMalloc(ctx, nbytes);
+        return _PyMem_DebugAlloc(0, ctx, nbytes);
 
     _PyMem_DebugCheckAddress(api->api_id, p);
     bumpserialno();