bpo-42536: GC track recycled tuples (GH-23623)
Several built-in and standard library types now ensure that their internal result tuples are always tracked by the garbage collector: - collections.OrderedDict.items - dict.items - enumerate - functools.reduce - itertools.combinations - itertools.combinations_with_replacement - itertools.permutations - itertools.product - itertools.zip_longest - zip Previously, they could have become untracked by a prior garbage collection.
This commit is contained in:
parent
2de5097ba4
commit
226a012d1c
|
@ -6,6 +6,7 @@ import builtins
|
||||||
import collections
|
import collections
|
||||||
import decimal
|
import decimal
|
||||||
import fractions
|
import fractions
|
||||||
|
import gc
|
||||||
import io
|
import io
|
||||||
import locale
|
import locale
|
||||||
import os
|
import os
|
||||||
|
@ -1756,6 +1757,18 @@ class BuiltinTest(unittest.TestCase):
|
||||||
l8 = self.iter_error(zip(Iter(3), "AB", strict=True), ValueError)
|
l8 = self.iter_error(zip(Iter(3), "AB", strict=True), ValueError)
|
||||||
self.assertEqual(l8, [(2, "A"), (1, "B")])
|
self.assertEqual(l8, [(2, "A"), (1, "B")])
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_zip_result_gc(self):
|
||||||
|
# bpo-42536: zip's tuple-reuse speed trick breaks the GC's assumptions
|
||||||
|
# about what can be untracked. Make sure we re-track result tuples
|
||||||
|
# whenever we reuse them.
|
||||||
|
it = zip([[]])
|
||||||
|
gc.collect()
|
||||||
|
# That GC collection probably untracked the recycled internal result
|
||||||
|
# tuple, which is initialized to (None,). Make sure it's re-tracked when
|
||||||
|
# it's mutated and returned from __next__:
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
def test_format(self):
|
def test_format(self):
|
||||||
# Test the basic machinery of the format() builtin. Don't test
|
# Test the basic machinery of the format() builtin. Don't test
|
||||||
# the specifics of the various formatters
|
# the specifics of the various formatters
|
||||||
|
|
|
@ -1452,6 +1452,25 @@ class DictTest(unittest.TestCase):
|
||||||
d = CustomReversedDict(pairs)
|
d = CustomReversedDict(pairs)
|
||||||
self.assertEqual(pairs[::-1], list(dict(d).items()))
|
self.assertEqual(pairs[::-1], list(dict(d).items()))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_dict_items_result_gc(self):
|
||||||
|
# bpo-42536: dict.items's tuple-reuse speed trick breaks the GC's
|
||||||
|
# assumptions about what can be untracked. Make sure we re-track result
|
||||||
|
# tuples whenever we reuse them.
|
||||||
|
it = iter({None: []}.items())
|
||||||
|
gc.collect()
|
||||||
|
# That GC collection probably untracked the recycled internal result
|
||||||
|
# tuple, which is initialized to (None, None). Make sure it's re-tracked
|
||||||
|
# when it's mutated and returned from __next__:
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_dict_items_result_gc(self):
|
||||||
|
# Same as test_dict_items_result_gc above, but reversed.
|
||||||
|
it = reversed({None: []}.items())
|
||||||
|
gc.collect()
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
|
||||||
class CAPITest(unittest.TestCase):
|
class CAPITest(unittest.TestCase):
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@ import unittest
|
||||||
import operator
|
import operator
|
||||||
import sys
|
import sys
|
||||||
import pickle
|
import pickle
|
||||||
|
import gc
|
||||||
|
|
||||||
from test import support
|
from test import support
|
||||||
|
|
||||||
|
@ -134,6 +135,18 @@ class EnumerateTestCase(unittest.TestCase, PickleTest):
|
||||||
self.assertEqual(len(set(map(id, list(enumerate(self.seq))))), len(self.seq))
|
self.assertEqual(len(set(map(id, list(enumerate(self.seq))))), len(self.seq))
|
||||||
self.assertEqual(len(set(map(id, enumerate(self.seq)))), min(1,len(self.seq)))
|
self.assertEqual(len(set(map(id, enumerate(self.seq)))), min(1,len(self.seq)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_enumerate_result_gc(self):
|
||||||
|
# bpo-42536: enumerate's tuple-reuse speed trick breaks the GC's
|
||||||
|
# assumptions about what can be untracked. Make sure we re-track result
|
||||||
|
# tuples whenever we reuse them.
|
||||||
|
it = self.enum([[]])
|
||||||
|
gc.collect()
|
||||||
|
# That GC collection probably untracked the recycled internal result
|
||||||
|
# tuple, which is initialized to (None, None). Make sure it's re-tracked
|
||||||
|
# when it's mutated and returned from __next__:
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
class MyEnum(enumerate):
|
class MyEnum(enumerate):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,8 @@ from functools import reduce
|
||||||
import sys
|
import sys
|
||||||
import struct
|
import struct
|
||||||
import threading
|
import threading
|
||||||
|
import gc
|
||||||
|
|
||||||
maxsize = support.MAX_Py_ssize_t
|
maxsize = support.MAX_Py_ssize_t
|
||||||
minsize = -maxsize-1
|
minsize = -maxsize-1
|
||||||
|
|
||||||
|
@ -1573,6 +1575,51 @@ class TestBasicOps(unittest.TestCase):
|
||||||
self.assertRaises(StopIteration, next, f(lambda x:x, []))
|
self.assertRaises(StopIteration, next, f(lambda x:x, []))
|
||||||
self.assertRaises(StopIteration, next, f(lambda x:x, StopNow()))
|
self.assertRaises(StopIteration, next, f(lambda x:x, StopNow()))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_combinations_result_gc(self):
|
||||||
|
# bpo-42536: combinations's tuple-reuse speed trick breaks the GC's
|
||||||
|
# assumptions about what can be untracked. Make sure we re-track result
|
||||||
|
# tuples whenever we reuse them.
|
||||||
|
it = combinations([None, []], 1)
|
||||||
|
next(it)
|
||||||
|
gc.collect()
|
||||||
|
# That GC collection probably untracked the recycled internal result
|
||||||
|
# tuple, which has the value (None,). Make sure it's re-tracked when
|
||||||
|
# it's mutated and returned from __next__:
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_combinations_with_replacement_result_gc(self):
|
||||||
|
# Ditto for combinations_with_replacement.
|
||||||
|
it = combinations_with_replacement([None, []], 1)
|
||||||
|
next(it)
|
||||||
|
gc.collect()
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_permutations_result_gc(self):
|
||||||
|
# Ditto for permutations.
|
||||||
|
it = permutations([None, []], 1)
|
||||||
|
next(it)
|
||||||
|
gc.collect()
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_product_result_gc(self):
|
||||||
|
# Ditto for product.
|
||||||
|
it = product([None, []])
|
||||||
|
next(it)
|
||||||
|
gc.collect()
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_zip_longest_result_gc(self):
|
||||||
|
# Ditto for zip_longest.
|
||||||
|
it = zip_longest([[]])
|
||||||
|
gc.collect()
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
|
|
||||||
class TestExamples(unittest.TestCase):
|
class TestExamples(unittest.TestCase):
|
||||||
|
|
||||||
def test_accumulate(self):
|
def test_accumulate(self):
|
||||||
|
|
|
@ -700,6 +700,17 @@ class OrderedDictTests:
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
a |= "BAD"
|
a |= "BAD"
|
||||||
|
|
||||||
|
@support.cpython_only
|
||||||
|
def test_ordered_dict_items_result_gc(self):
|
||||||
|
# bpo-42536: OrderedDict.items's tuple-reuse speed trick breaks the GC's
|
||||||
|
# assumptions about what can be untracked. Make sure we re-track result
|
||||||
|
# tuples whenever we reuse them.
|
||||||
|
it = iter(self.OrderedDict({None: []}).items())
|
||||||
|
gc.collect()
|
||||||
|
# That GC collection probably untracked the recycled internal result
|
||||||
|
# tuple, which is initialized to (None, None). Make sure it's re-tracked
|
||||||
|
# when it's mutated and returned from __next__:
|
||||||
|
self.assertTrue(gc.is_tracked(next(it)))
|
||||||
|
|
||||||
class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase):
|
class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase):
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
Several built-in and standard library types now ensure that their internal
|
||||||
|
result tuples are always tracked by the :term:`garbage collector
|
||||||
|
<garbage collection>`:
|
||||||
|
|
||||||
|
- :meth:`collections.OrderedDict.items() <collections.OrderedDict>`
|
||||||
|
|
||||||
|
- :meth:`dict.items`
|
||||||
|
|
||||||
|
- :func:`enumerate`
|
||||||
|
|
||||||
|
- :func:`functools.reduce`
|
||||||
|
|
||||||
|
- :func:`itertools.combinations`
|
||||||
|
|
||||||
|
- :func:`itertools.combinations_with_replacement`
|
||||||
|
|
||||||
|
- :func:`itertools.permutations`
|
||||||
|
|
||||||
|
- :func:`itertools.product`
|
||||||
|
|
||||||
|
- :func:`itertools.zip_longest`
|
||||||
|
|
||||||
|
- :func:`zip`
|
||||||
|
|
||||||
|
Previously, they could have become untracked by a prior garbage collection.
|
||||||
|
Patch by Brandt Bucher.
|
|
@ -1,5 +1,6 @@
|
||||||
#include "Python.h"
|
#include "Python.h"
|
||||||
#include "pycore_long.h" // _PyLong_GetZero()
|
#include "pycore_long.h" // _PyLong_GetZero()
|
||||||
|
#include "pycore_object.h" // _PyObject_GC_TRACK
|
||||||
#include "pycore_pystate.h" // _PyThreadState_GET()
|
#include "pycore_pystate.h" // _PyThreadState_GET()
|
||||||
#include "pycore_tuple.h" // _PyTuple_ITEMS()
|
#include "pycore_tuple.h" // _PyTuple_ITEMS()
|
||||||
#include "structmember.h" // PyMemberDef
|
#include "structmember.h" // PyMemberDef
|
||||||
|
@ -673,6 +674,11 @@ functools_reduce(PyObject *self, PyObject *args)
|
||||||
if ((result = PyObject_Call(func, args, NULL)) == NULL) {
|
if ((result = PyObject_Call(func, args, NULL)) == NULL) {
|
||||||
goto Fail;
|
goto Fail;
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this args tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(args)) {
|
||||||
|
_PyObject_GC_TRACK(args);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#define PY_SSIZE_T_CLEAN
|
#define PY_SSIZE_T_CLEAN
|
||||||
#include "Python.h"
|
#include "Python.h"
|
||||||
#include "pycore_long.h" // _PyLong_GetZero()
|
#include "pycore_long.h" // _PyLong_GetZero()
|
||||||
|
#include "pycore_object.h" // _PyObject_GC_TRACK()
|
||||||
#include "pycore_tuple.h" // _PyTuple_ITEMS()
|
#include "pycore_tuple.h" // _PyTuple_ITEMS()
|
||||||
#include <stddef.h> // offsetof()
|
#include <stddef.h> // offsetof()
|
||||||
|
|
||||||
|
@ -2378,6 +2379,11 @@ product_next(productobject *lz)
|
||||||
lz->result = result;
|
lz->result = result;
|
||||||
Py_DECREF(old_result);
|
Py_DECREF(old_result);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
else if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
/* Now, we've got the only copy so we can update it in-place */
|
/* Now, we've got the only copy so we can update it in-place */
|
||||||
assert (npools==0 || Py_REFCNT(result) == 1);
|
assert (npools==0 || Py_REFCNT(result) == 1);
|
||||||
|
|
||||||
|
@ -2701,6 +2707,11 @@ combinations_next(combinationsobject *co)
|
||||||
co->result = result;
|
co->result = result;
|
||||||
Py_DECREF(old_result);
|
Py_DECREF(old_result);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
else if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
/* Now, we've got the only copy so we can update it in-place
|
/* Now, we've got the only copy so we can update it in-place
|
||||||
* CPython's empty tuple is a singleton and cached in
|
* CPython's empty tuple is a singleton and cached in
|
||||||
* PyTuple's freelist.
|
* PyTuple's freelist.
|
||||||
|
@ -3035,6 +3046,11 @@ cwr_next(cwrobject *co)
|
||||||
co->result = result;
|
co->result = result;
|
||||||
Py_DECREF(old_result);
|
Py_DECREF(old_result);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
else if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
/* Now, we've got the only copy so we can update it in-place CPython's
|
/* Now, we've got the only copy so we can update it in-place CPython's
|
||||||
empty tuple is a singleton and cached in PyTuple's freelist. */
|
empty tuple is a singleton and cached in PyTuple's freelist. */
|
||||||
assert(r == 0 || Py_REFCNT(result) == 1);
|
assert(r == 0 || Py_REFCNT(result) == 1);
|
||||||
|
@ -3379,6 +3395,11 @@ permutations_next(permutationsobject *po)
|
||||||
po->result = result;
|
po->result = result;
|
||||||
Py_DECREF(old_result);
|
Py_DECREF(old_result);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
else if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
/* Now, we've got the only copy so we can update it in-place */
|
/* Now, we've got the only copy so we can update it in-place */
|
||||||
assert(r == 0 || Py_REFCNT(result) == 1);
|
assert(r == 0 || Py_REFCNT(result) == 1);
|
||||||
|
|
||||||
|
@ -4649,6 +4670,11 @@ zip_longest_next(ziplongestobject *lz)
|
||||||
PyTuple_SET_ITEM(result, i, item);
|
PyTuple_SET_ITEM(result, i, item);
|
||||||
Py_DECREF(olditem);
|
Py_DECREF(olditem);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
result = PyTuple_New(tuplesize);
|
result = PyTuple_New(tuplesize);
|
||||||
if (result == NULL)
|
if (result == NULL)
|
||||||
|
|
|
@ -3989,6 +3989,11 @@ dictiter_iternextitem(dictiterobject *di)
|
||||||
Py_INCREF(result);
|
Py_INCREF(result);
|
||||||
Py_DECREF(oldkey);
|
Py_DECREF(oldkey);
|
||||||
Py_DECREF(oldvalue);
|
Py_DECREF(oldvalue);
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
result = PyTuple_New(2);
|
result = PyTuple_New(2);
|
||||||
|
@ -4104,6 +4109,11 @@ dictreviter_iternext(dictiterobject *di)
|
||||||
Py_INCREF(result);
|
Py_INCREF(result);
|
||||||
Py_DECREF(oldkey);
|
Py_DECREF(oldkey);
|
||||||
Py_DECREF(oldvalue);
|
Py_DECREF(oldvalue);
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since
|
||||||
|
// we're recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
result = PyTuple_New(2);
|
result = PyTuple_New(2);
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
#include "Python.h"
|
#include "Python.h"
|
||||||
#include "pycore_long.h" // _PyLong_GetOne()
|
#include "pycore_long.h" // _PyLong_GetOne()
|
||||||
|
#include "pycore_object.h" // _PyObject_GC_TRACK()
|
||||||
|
|
||||||
#include "clinic/enumobject.c.h"
|
#include "clinic/enumobject.c.h"
|
||||||
|
|
||||||
|
@ -131,6 +132,11 @@ enum_next_long(enumobject *en, PyObject* next_item)
|
||||||
PyTuple_SET_ITEM(result, 1, next_item);
|
PyTuple_SET_ITEM(result, 1, next_item);
|
||||||
Py_DECREF(old_index);
|
Py_DECREF(old_index);
|
||||||
Py_DECREF(old_item);
|
Py_DECREF(old_item);
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
result = PyTuple_New(2);
|
result = PyTuple_New(2);
|
||||||
|
@ -176,6 +182,11 @@ enum_next(enumobject *en)
|
||||||
PyTuple_SET_ITEM(result, 1, next_item);
|
PyTuple_SET_ITEM(result, 1, next_item);
|
||||||
Py_DECREF(old_index);
|
Py_DECREF(old_index);
|
||||||
Py_DECREF(old_item);
|
Py_DECREF(old_item);
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
result = PyTuple_New(2);
|
result = PyTuple_New(2);
|
||||||
|
|
|
@ -1814,6 +1814,11 @@ odictiter_iternext(odictiterobject *di)
|
||||||
Py_INCREF(result);
|
Py_INCREF(result);
|
||||||
Py_DECREF(PyTuple_GET_ITEM(result, 0)); /* borrowed */
|
Py_DECREF(PyTuple_GET_ITEM(result, 0)); /* borrowed */
|
||||||
Py_DECREF(PyTuple_GET_ITEM(result, 1)); /* borrowed */
|
Py_DECREF(PyTuple_GET_ITEM(result, 1)); /* borrowed */
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
result = PyTuple_New(2);
|
result = PyTuple_New(2);
|
||||||
|
|
|
@ -2636,6 +2636,11 @@ zip_next(zipobject *lz)
|
||||||
PyTuple_SET_ITEM(result, i, item);
|
PyTuple_SET_ITEM(result, i, item);
|
||||||
Py_DECREF(olditem);
|
Py_DECREF(olditem);
|
||||||
}
|
}
|
||||||
|
// bpo-42536: The GC may have untracked this result tuple. Since we're
|
||||||
|
// recycling it, make sure it's tracked again:
|
||||||
|
if (!_PyObject_GC_IS_TRACKED(result)) {
|
||||||
|
_PyObject_GC_TRACK(result);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
result = PyTuple_New(tuplesize);
|
result = PyTuple_New(tuplesize);
|
||||||
if (result == NULL)
|
if (result == NULL)
|
||||||
|
|
Loading…
Reference in New Issue