From 8cd1dbae32d9303caac3a473d3332f17bc98c921 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 24 Oct 2020 21:14:23 +0300 Subject: [PATCH] bpo-41052: Fix pickling heap types implemented in C with protocols 0 and 1 (GH-22870) --- Lib/copyreg.py | 4 +++ Lib/test/pickletester.py | 18 ++++++++++ .../2020-10-21-23-45-02.bpo-41052.3N7J2J.rst | 2 ++ Modules/_bz2module.c | 32 ----------------- Modules/_lzmamodule.c | 30 ---------------- Modules/_randommodule.c | 17 --------- Modules/clinic/_bz2module.c.h | 36 +------------------ Modules/clinic/_lzmamodule.c.h | 36 +------------------ Modules/clinic/_randommodule.c.h | 19 +--------- 9 files changed, 27 insertions(+), 167 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst diff --git a/Lib/copyreg.py b/Lib/copyreg.py index dfc463c49a3..7ab8c128eb0 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -48,6 +48,7 @@ def _reconstructor(cls, base, state): return obj _HEAPTYPE = 1<<9 +_new_type = type(int.__new__) # Python code for object.__reduce_ex__ for protocols 0 and 1 @@ -57,6 +58,9 @@ def _reduce_ex(self, proto): for base in cls.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break + new = base.__new__ + if isinstance(new, _new_type) and new.__self__ is base: + break else: base = object # not really reachable if base is object: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index fb972a3ba5e..ae288f5d012 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1969,6 +1969,17 @@ class AbstractPickleTests(unittest.TestCase): self.assertEqual(B(x), B(y), detail) self.assertEqual(x.__dict__, y.__dict__, detail) + def test_newobj_overridden_new(self): + # Test that Python class with C implemented __new__ is pickleable + for proto in protocols: + x = MyIntWithNew2(1) + x.foo = 42 + s = self.dumps(x, proto) + y = self.loads(s) + self.assertIs(type(y), MyIntWithNew2) + self.assertEqual(int(y), 1) + self.assertEqual(y.foo, 42) + def test_newobj_not_class(self): # Issue 24552 global SimpleNewObj @@ -3089,6 +3100,13 @@ myclasses = [MyInt, MyFloat, MyStr, MyUnicode, MyTuple, MyList, MyDict, MySet, MyFrozenSet] +class MyIntWithNew(int): + def __new__(cls, value): + raise AssertionError + +class MyIntWithNew2(MyIntWithNew): + __new__ = int.__new__ + class SlotList(MyList): __slots__ = ["foo"] diff --git a/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst b/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst new file mode 100644 index 00000000000..528e90ed134 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst @@ -0,0 +1,2 @@ +Pickling heap types implemented in C with protocols 0 and 1 raises now an +error instead of producing incorrect data. diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index effb0de2e65..bfcdac69246 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -272,21 +272,6 @@ _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self) return result; } -/*[clinic input] -_bz2.BZ2Compressor.__reduce__ - -[clinic start generated code]*/ - -static PyObject * -_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self) -/*[clinic end generated code: output=d13db66ae043e141 input=e09bccef0e6731b2]*/ -{ - PyErr_Format(PyExc_TypeError, - "cannot pickle %s object", - Py_TYPE(self)->tp_name); - return NULL; -} - static void* BZ2_Malloc(void* ctx, int items, int size) { @@ -399,7 +384,6 @@ BZ2Compressor_traverse(BZ2Compressor *self, visitproc visit, void *arg) static PyMethodDef BZ2Compressor_methods[] = { _BZ2_BZ2COMPRESSOR_COMPRESS_METHODDEF _BZ2_BZ2COMPRESSOR_FLUSH_METHODDEF - _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF {NULL} }; @@ -642,21 +626,6 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, return result; } -/*[clinic input] -_bz2.BZ2Decompressor.__reduce__ - -[clinic start generated code]*/ - -static PyObject * -_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self) -/*[clinic end generated code: output=f6a40650813f482e input=8db9175a609fdd43]*/ -{ - PyErr_Format(PyExc_TypeError, - "cannot pickle %s object", - Py_TYPE(self)->tp_name); - return NULL; -} - /* Argument Clinic is not used since the Argument Clinic always want to check the type which would be wrong here */ static int @@ -746,7 +715,6 @@ BZ2Decompressor_traverse(BZ2Decompressor *self, visitproc visit, void *arg) static PyMethodDef BZ2Decompressor_methods[] = { _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF - _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF {NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 24e1d6c2884..b01f6300098 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -825,24 +825,9 @@ Compressor_dealloc(Compressor *self) Py_DECREF(tp); } -/*[clinic input] -_lzma.LZMACompressor.__reduce__ -[clinic start generated code]*/ - -static PyObject * -_lzma_LZMACompressor___reduce___impl(Compressor *self) -/*[clinic end generated code: output=b49a0538d1cad752 input=6be52aba16b513c1]*/ -{ - PyErr_Format(PyExc_TypeError, - "cannot pickle %s object", - Py_TYPE(self)->tp_name); - return NULL; -} - static PyMethodDef Compressor_methods[] = { _LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF _LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF - _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF {NULL} }; @@ -1309,23 +1294,8 @@ Decompressor_traverse(Decompressor *self, visitproc visit, void *arg) return 0; } -/*[clinic input] -_lzma.LZMADecompressor.__reduce__ -[clinic start generated code]*/ - -static PyObject * -_lzma_LZMADecompressor___reduce___impl(Decompressor *self) -/*[clinic end generated code: output=2611fff0104a9c30 input=b9882e030aecd9a5]*/ -{ - PyErr_Format(PyExc_TypeError, - "cannot pickle %s object", - Py_TYPE(self)->tp_name); - return NULL; -} - static PyMethodDef Decompressor_methods[] = { _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF - _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF {NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index b8bc0449c1b..d3387080862 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -536,29 +536,12 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } -/*[clinic input] - -_random.Random.__reduce__ - -[clinic start generated code]*/ - -static PyObject * -_random_Random___reduce___impl(RandomObject *self) -/*[clinic end generated code: output=ddea0dcdb60ffd6d input=bd38ec35fd157e0f]*/ -{ - PyErr_Format(PyExc_TypeError, - "cannot pickle %s object", - Py_TYPE(self)->tp_name); - return NULL; -} - static PyMethodDef random_methods[] = { _RANDOM_RANDOM_RANDOM_METHODDEF _RANDOM_RANDOM_SEED_METHODDEF _RANDOM_RANDOM_GETSTATE_METHODDEF _RANDOM_RANDOM_SETSTATE_METHODDEF _RANDOM_RANDOM_GETRANDBITS_METHODDEF - _RANDOM_RANDOM___REDUCE___METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index ff67d34155d..71ad0b13571 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -65,23 +65,6 @@ _bz2_BZ2Compressor_flush(BZ2Compressor *self, PyObject *Py_UNUSED(ignored)) return _bz2_BZ2Compressor_flush_impl(self); } -PyDoc_STRVAR(_bz2_BZ2Compressor___reduce____doc__, -"__reduce__($self, /)\n" -"--\n" -"\n"); - -#define _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF \ - {"__reduce__", (PyCFunction)_bz2_BZ2Compressor___reduce__, METH_NOARGS, _bz2_BZ2Compressor___reduce____doc__}, - -static PyObject * -_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self); - -static PyObject * -_bz2_BZ2Compressor___reduce__(BZ2Compressor *self, PyObject *Py_UNUSED(ignored)) -{ - return _bz2_BZ2Compressor___reduce___impl(self); -} - PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" @@ -156,21 +139,4 @@ exit: return return_value; } - -PyDoc_STRVAR(_bz2_BZ2Decompressor___reduce____doc__, -"__reduce__($self, /)\n" -"--\n" -"\n"); - -#define _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF \ - {"__reduce__", (PyCFunction)_bz2_BZ2Decompressor___reduce__, METH_NOARGS, _bz2_BZ2Decompressor___reduce____doc__}, - -static PyObject * -_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self); - -static PyObject * -_bz2_BZ2Decompressor___reduce__(BZ2Decompressor *self, PyObject *Py_UNUSED(ignored)) -{ - return _bz2_BZ2Decompressor___reduce___impl(self); -} -/*[clinic end generated code: output=001f31fdacb4cb01 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed10705d7a9fd598 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index e15cc0c7e74..526031ade3b 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -65,23 +65,6 @@ _lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored)) return _lzma_LZMACompressor_flush_impl(self); } -PyDoc_STRVAR(_lzma_LZMACompressor___reduce____doc__, -"__reduce__($self, /)\n" -"--\n" -"\n"); - -#define _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF \ - {"__reduce__", (PyCFunction)_lzma_LZMACompressor___reduce__, METH_NOARGS, _lzma_LZMACompressor___reduce____doc__}, - -static PyObject * -_lzma_LZMACompressor___reduce___impl(Compressor *self); - -static PyObject * -_lzma_LZMACompressor___reduce__(Compressor *self, PyObject *Py_UNUSED(ignored)) -{ - return _lzma_LZMACompressor___reduce___impl(self); -} - PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" @@ -228,23 +211,6 @@ exit: return return_value; } -PyDoc_STRVAR(_lzma_LZMADecompressor___reduce____doc__, -"__reduce__($self, /)\n" -"--\n" -"\n"); - -#define _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF \ - {"__reduce__", (PyCFunction)_lzma_LZMADecompressor___reduce__, METH_NOARGS, _lzma_LZMADecompressor___reduce____doc__}, - -static PyObject * -_lzma_LZMADecompressor___reduce___impl(Decompressor *self); - -static PyObject * -_lzma_LZMADecompressor___reduce__(Decompressor *self, PyObject *Py_UNUSED(ignored)) -{ - return _lzma_LZMADecompressor___reduce___impl(self); -} - PyDoc_STRVAR(_lzma_is_check_supported__doc__, "is_check_supported($module, check_id, /)\n" "--\n" @@ -320,4 +286,4 @@ exit: return return_value; } -/*[clinic end generated code: output=d89b6159e98544be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=867b9e334053b679 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 3322a370288..b3cd435b6f2 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -109,21 +109,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg) exit: return return_value; } - -PyDoc_STRVAR(_random_Random___reduce____doc__, -"__reduce__($self, /)\n" -"--\n" -"\n"); - -#define _RANDOM_RANDOM___REDUCE___METHODDEF \ - {"__reduce__", (PyCFunction)_random_Random___reduce__, METH_NOARGS, _random_Random___reduce____doc__}, - -static PyObject * -_random_Random___reduce___impl(RandomObject *self); - -static PyObject * -_random_Random___reduce__(RandomObject *self, PyObject *Py_UNUSED(ignored)) -{ - return _random_Random___reduce___impl(self); -} -/*[clinic end generated code: output=450f0961c2c92389 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cc8a23b2757dc6ba input=a9049054013a1b77]*/