[3.9] bpo-41052: Fix pickling heap types implemented in C with protocols 0 and 1 (GH-22870). (GH-22963)

(cherry picked from commit 8cd1dbae32)

Lib/copyreg.py

@@ -48,6 +48,7 @@ def _reconstructor(cls, base, state):
     return obj
 
 _HEAPTYPE = 1<<9
+_new_type = type(int.__new__)
 
 # Python code for object.__reduce_ex__ for protocols 0 and 1
 
@@ -57,6 +58,9 @@ def _reduce_ex(self, proto):
     for base in cls.__mro__:
         if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
             break
+        new = base.__new__
+        if isinstance(new, _new_type) and new.__self__ is base:
+            break
     else:
         base = object # not really reachable
     if base is object:

Lib/test/pickletester.py

@@ -1965,6 +1965,17 @@ class AbstractPickleTests(unittest.TestCase):
                 self.assertEqual(B(x), B(y), detail)
                 self.assertEqual(x.__dict__, y.__dict__, detail)
 
+    def test_newobj_overridden_new(self):
+        # Test that Python class with C implemented __new__ is pickleable
+        for proto in protocols:
+            x = MyIntWithNew2(1)
+            x.foo = 42
+            s = self.dumps(x, proto)
+            y = self.loads(s)
+            self.assertIs(type(y), MyIntWithNew2)
+            self.assertEqual(int(y), 1)
+            self.assertEqual(y.foo, 42)
+
     def test_newobj_not_class(self):
         # Issue 24552
         global SimpleNewObj
@@ -3085,6 +3096,13 @@ myclasses = [MyInt, MyFloat,
              MyStr, MyUnicode,
              MyTuple, MyList, MyDict, MySet, MyFrozenSet]
 
+class MyIntWithNew(int):
+    def __new__(cls, value):
+        raise AssertionError
+
+class MyIntWithNew2(MyIntWithNew):
+    __new__ = int.__new__
+
 
 class SlotList(MyList):
     __slots__ = ["foo"]

Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst

@@ -0,0 +1,2 @@
+Pickling heap types implemented in C with protocols 0 and 1 raises now an
+error instead of producing incorrect data.

Modules/_randommodule.c

@@ -537,29 +537,12 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 }
 
 
-/*[clinic input]
-
-_random.Random.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self)
-/*[clinic end generated code: output=ddea0dcdb60ffd6d input=bd38ec35fd157e0f]*/
-{
-    PyErr_Format(PyExc_TypeError,
-                 "cannot pickle %s object",
-                 Py_TYPE(self)->tp_name);
-    return NULL;
-}
-
 static PyMethodDef random_methods[] = {
     _RANDOM_RANDOM_RANDOM_METHODDEF
     _RANDOM_RANDOM_SEED_METHODDEF
     _RANDOM_RANDOM_GETSTATE_METHODDEF
     _RANDOM_RANDOM_SETSTATE_METHODDEF
     _RANDOM_RANDOM_GETRANDBITS_METHODDEF
-    _RANDOM_RANDOM___REDUCE___METHODDEF
     {NULL,              NULL}           /* sentinel */
 };
 

Modules/clinic/_randommodule.c.h

@@ -114,21 +114,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg)
 exit:
     return return_value;
 }
-
+/*[clinic end generated code: output=a7feb0c9c8d1b627 input=a9049054013a1b77]*/
-PyDoc_STRVAR(_random_Random___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _RANDOM_RANDOM___REDUCE___METHODDEF    \
-    {"__reduce__", (PyCFunction)_random_Random___reduce__, METH_NOARGS, _random_Random___reduce____doc__},
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self);
-
-static PyObject *
-_random_Random___reduce__(RandomObject *self, PyObject *Py_UNUSED(ignored))
-{
-    return _random_Random___reduce___impl(self);
-}
-/*[clinic end generated code: output=d8a99be3f1192219 input=a9049054013a1b77]*/