bpo-41052: Fix pickling heap types implemented in C with protocols 0 and 1 (GH-22870)

Lib/copyreg.py

@@ -48,6 +48,7 @@ def _reconstructor(cls, base, state):
     return obj
 
 _HEAPTYPE = 1<<9
+_new_type = type(int.__new__)
 
 # Python code for object.__reduce_ex__ for protocols 0 and 1
 
@@ -57,6 +58,9 @@ def _reduce_ex(self, proto):
     for base in cls.__mro__:
         if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
             break
+        new = base.__new__
+        if isinstance(new, _new_type) and new.__self__ is base:
+            break
     else:
         base = object # not really reachable
     if base is object:

Lib/test/pickletester.py

@@ -1969,6 +1969,17 @@ class AbstractPickleTests(unittest.TestCase):
                 self.assertEqual(B(x), B(y), detail)
                 self.assertEqual(x.__dict__, y.__dict__, detail)
 
+    def test_newobj_overridden_new(self):
+        # Test that Python class with C implemented __new__ is pickleable
+        for proto in protocols:
+            x = MyIntWithNew2(1)
+            x.foo = 42
+            s = self.dumps(x, proto)
+            y = self.loads(s)
+            self.assertIs(type(y), MyIntWithNew2)
+            self.assertEqual(int(y), 1)
+            self.assertEqual(y.foo, 42)
+
     def test_newobj_not_class(self):
         # Issue 24552
         global SimpleNewObj
@@ -3089,6 +3100,13 @@ myclasses = [MyInt, MyFloat,
              MyStr, MyUnicode,
              MyTuple, MyList, MyDict, MySet, MyFrozenSet]
 
+class MyIntWithNew(int):
+    def __new__(cls, value):
+        raise AssertionError
+
+class MyIntWithNew2(MyIntWithNew):
+    __new__ = int.__new__
+
 
 class SlotList(MyList):
     __slots__ = ["foo"]

Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst

@@ -0,0 +1,2 @@
+Pickling heap types implemented in C with protocols 0 and 1 raises now an
+error instead of producing incorrect data.

Modules/_bz2module.c

@@ -272,21 +272,6 @@ _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self)
     return result;
 }
 
-/*[clinic input]
-_bz2.BZ2Compressor.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self)
-/*[clinic end generated code: output=d13db66ae043e141 input=e09bccef0e6731b2]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 static void*
 BZ2_Malloc(void* ctx, int items, int size)
 {
@@ -399,7 +384,6 @@ BZ2Compressor_traverse(BZ2Compressor *self, visitproc visit, void *arg)
 static PyMethodDef BZ2Compressor_methods[] = {
     _BZ2_BZ2COMPRESSOR_COMPRESS_METHODDEF
     _BZ2_BZ2COMPRESSOR_FLUSH_METHODDEF
-    _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF
     {NULL}
 };
 
@@ -642,21 +626,6 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data,
     return result;
 }
 
-/*[clinic input]
-_bz2.BZ2Decompressor.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self)
-/*[clinic end generated code: output=f6a40650813f482e input=8db9175a609fdd43]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 /* Argument Clinic is not used since the Argument Clinic always want to
    check the type which would be wrong here */
 static int
@@ -746,7 +715,6 @@ BZ2Decompressor_traverse(BZ2Decompressor *self, visitproc visit, void *arg)
 
 static PyMethodDef BZ2Decompressor_methods[] = {
     _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF
-    _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF
     {NULL}
 };
 

Modules/_lzmamodule.c

@@ -825,24 +825,9 @@ Compressor_dealloc(Compressor *self)
     Py_DECREF(tp);
 }
 
-/*[clinic input]
-_lzma.LZMACompressor.__reduce__
-[clinic start generated code]*/
-
-static PyObject *
-_lzma_LZMACompressor___reduce___impl(Compressor *self)
-/*[clinic end generated code: output=b49a0538d1cad752 input=6be52aba16b513c1]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 static PyMethodDef Compressor_methods[] = {
     _LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF
     _LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF
-    _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF
     {NULL}
 };
 
@@ -1309,23 +1294,8 @@ Decompressor_traverse(Decompressor *self, visitproc visit, void *arg)
     return 0;
 }
 
-/*[clinic input]
-_lzma.LZMADecompressor.__reduce__
-[clinic start generated code]*/
-
-static PyObject *
-_lzma_LZMADecompressor___reduce___impl(Decompressor *self)
-/*[clinic end generated code: output=2611fff0104a9c30 input=b9882e030aecd9a5]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 static PyMethodDef Decompressor_methods[] = {
     _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF
-    _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF
     {NULL}
 };
 

Modules/_randommodule.c

@@ -536,29 +536,12 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 }
 
 
-/*[clinic input]
-
-_random.Random.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self)
-/*[clinic end generated code: output=ddea0dcdb60ffd6d input=bd38ec35fd157e0f]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 static PyMethodDef random_methods[] = {
     _RANDOM_RANDOM_RANDOM_METHODDEF
     _RANDOM_RANDOM_SEED_METHODDEF
     _RANDOM_RANDOM_GETSTATE_METHODDEF
     _RANDOM_RANDOM_SETSTATE_METHODDEF
     _RANDOM_RANDOM_GETRANDBITS_METHODDEF
-    _RANDOM_RANDOM___REDUCE___METHODDEF
     {NULL,              NULL}           /* sentinel */
 };
 

Modules/clinic/_bz2module.c.h

@@ -65,23 +65,6 @@ _bz2_BZ2Compressor_flush(BZ2Compressor *self, PyObject *Py_UNUSED(ignored))
     return _bz2_BZ2Compressor_flush_impl(self);
 }
 
-PyDoc_STRVAR(_bz2_BZ2Compressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_bz2_BZ2Compressor___reduce__, METH_NOARGS, _bz2_BZ2Compressor___reduce____doc__},
-
-static PyObject *
-_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self);
-
-static PyObject *
-_bz2_BZ2Compressor___reduce__(BZ2Compressor *self, PyObject *Py_UNUSED(ignored))
-{
-    return _bz2_BZ2Compressor___reduce___impl(self);
-}
-
 PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__,
 "decompress($self, /, data, max_length=-1)\n"
 "--\n"
@@ -156,21 +139,4 @@ exit:
 
     return return_value;
 }
-
+/*[clinic end generated code: output=ed10705d7a9fd598 input=a9049054013a1b77]*/
-PyDoc_STRVAR(_bz2_BZ2Decompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_bz2_BZ2Decompressor___reduce__, METH_NOARGS, _bz2_BZ2Decompressor___reduce____doc__},
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self);
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce__(BZ2Decompressor *self, PyObject *Py_UNUSED(ignored))
-{
-    return _bz2_BZ2Decompressor___reduce___impl(self);
-}
-/*[clinic end generated code: output=001f31fdacb4cb01 input=a9049054013a1b77]*/

Modules/clinic/_lzmamodule.c.h

@@ -65,23 +65,6 @@ _lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored))
     return _lzma_LZMACompressor_flush_impl(self);
 }
 
-PyDoc_STRVAR(_lzma_LZMACompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_lzma_LZMACompressor___reduce__, METH_NOARGS, _lzma_LZMACompressor___reduce____doc__},
-
-static PyObject *
-_lzma_LZMACompressor___reduce___impl(Compressor *self);
-
-static PyObject *
-_lzma_LZMACompressor___reduce__(Compressor *self, PyObject *Py_UNUSED(ignored))
-{
-    return _lzma_LZMACompressor___reduce___impl(self);
-}
-
 PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__,
 "decompress($self, /, data, max_length=-1)\n"
 "--\n"
@@ -228,23 +211,6 @@ exit:
     return return_value;
 }
 
-PyDoc_STRVAR(_lzma_LZMADecompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_lzma_LZMADecompressor___reduce__, METH_NOARGS, _lzma_LZMADecompressor___reduce____doc__},
-
-static PyObject *
-_lzma_LZMADecompressor___reduce___impl(Decompressor *self);
-
-static PyObject *
-_lzma_LZMADecompressor___reduce__(Decompressor *self, PyObject *Py_UNUSED(ignored))
-{
-    return _lzma_LZMADecompressor___reduce___impl(self);
-}
-
 PyDoc_STRVAR(_lzma_is_check_supported__doc__,
 "is_check_supported($module, check_id, /)\n"
 "--\n"
@@ -320,4 +286,4 @@ exit:
 
     return return_value;
 }
-/*[clinic end generated code: output=d89b6159e98544be input=a9049054013a1b77]*/
+/*[clinic end generated code: output=867b9e334053b679 input=a9049054013a1b77]*/

Modules/clinic/_randommodule.c.h

@@ -109,21 +109,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-
+/*[clinic end generated code: output=cc8a23b2757dc6ba input=a9049054013a1b77]*/
-PyDoc_STRVAR(_random_Random___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _RANDOM_RANDOM___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_random_Random___reduce__, METH_NOARGS, _random_Random___reduce____doc__},
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self);
-
-static PyObject *
-_random_Random___reduce__(RandomObject *self, PyObject *Py_UNUSED(ignored))
-{
-    return _random_Random___reduce___impl(self);
-}
-/*[clinic end generated code: output=450f0961c2c92389 input=a9049054013a1b77]*/