Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-112087: Make __sizeof__ and listiter_{len, next} to be threadsafe (gh-114843)

This commit is contained in:
Donghee Na 2024-02-15 02:00:50 +09:00 committed by GitHub
parent 4b2d1786cc
commit a2d4281415
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 80 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Lib/test/support/__init__.py
View File

@ -1727,6 +1727,8 @@ def _check_tracemalloc():
def check_free_after_iterating(test, iter, cls, args=()):
done = False
def wrapper():
class A(cls):
def __del__(self):
nonlocal done
@ -1736,10 +1738,11 @@ def check_free_after_iterating(test, iter, cls, args=()):
except StopIteration:
pass
done = False
it = iter(A(*args))
# Issue 26494: Shouldn't crash
test.assertRaises(StopIteration, next, it)
wrapper()
# The sequence should be deallocated just after the end of iterating
gc_collect()
test.assertTrue(done)

2
Lib/test/test_iter.py
View File

@ -302,7 +302,7 @@ class TestCase(unittest.TestCase):
# listiter_reduce_general
self.assertEqual(
run("reversed", orig["reversed"](list(range(8)))),
(iter, ([],))
(reversed, ([],))
)
for case in types:

2
Misc/NEWS.d/next/Core and Builtins/2024-02-14-23-50-55.gh-issue-112087.H_4W_v.rst Normal file
View File

@ -0,0 +1,2 @@
For an empty reverse iterator for list will be reduced to :func:`reversed`.
Patch by Donghee Na

96
Objects/listobject.c
View File

@ -20,6 +20,14 @@ class list "PyListObject *" "&PyList_Type"
_Py_DECLARE_STR(list_err, "list index out of range");
#ifdef Py_GIL_DISABLED
# define LOAD_SSIZE(value) _Py_atomic_load_ssize_relaxed(&value)
# define STORE_SSIZE(value, new_value) _Py_atomic_store_ssize_relaxed(&value, new_value)
#else
# define LOAD_SSIZE(value) value
# define STORE_SSIZE(value, new_value) value = new_value
#endif
#ifdef WITH_FREELISTS
static struct _Py_list_freelist *
get_list_freelist(void)
@ -2971,7 +2979,8 @@ list___sizeof___impl(PyListObject *self)
/*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/
{
size_t res = _PyObject_SIZE(Py_TYPE(self));
res += (size_t)self->allocated * sizeof(void*);
Py_ssize_t allocated = LOAD_SSIZE(self->allocated);
res += (size_t)allocated * sizeof(void*);
return PyLong_FromSize_t(res);
}
@ -3373,33 +3382,34 @@ static PyObject *
listiter_next(PyObject *self)
{
_PyListIterObject *it = (_PyListIterObject *)self;
PyListObject *seq;
PyObject *item;
assert(it != NULL);
seq = it->it_seq;
if (seq == NULL)
Py_ssize_t index = LOAD_SSIZE(it->it_index);
if (index < 0) {
return NULL;
assert(PyList_Check(seq));
if (it->it_index < PyList_GET_SIZE(seq)) {
item = PyList_GET_ITEM(seq, it->it_index);
++it->it_index;
return Py_NewRef(item);
}
PyObject *item = list_get_item_ref(it->it_seq, index);
if (item == NULL) {
// out-of-bounds
STORE_SSIZE(it->it_index, -1);
#ifndef Py_GIL_DISABLED
PyListObject *seq = it->it_seq;
it->it_seq = NULL;
Py_DECREF(seq);
#endif
return NULL;
}
STORE_SSIZE(it->it_index, index + 1);
return item;
}
static PyObject *
listiter_len(PyObject *self, PyObject *Py_UNUSED(ignored))
{
assert(self != NULL);
_PyListIterObject *it = (_PyListIterObject *)self;
Py_ssize_t len;
if (it->it_seq) {
len = PyList_GET_SIZE(it->it_seq) - it->it_index;
Py_ssize_t index = LOAD_SSIZE(it->it_index);
if (index >= 0) {
Py_ssize_t len = PyList_GET_SIZE(it->it_seq) - index;
if (len >= 0)
return PyLong_FromSsize_t(len);
}
@ -3420,8 +3430,8 @@ listiter_setstate(PyObject *self, PyObject *state)
if (index == -1 && PyErr_Occurred())
return NULL;
if (it->it_seq != NULL) {
if (index < 0)
index = 0;
if (index < -1)
index = -1;
else if (index > PyList_GET_SIZE(it->it_seq))
index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */
it->it_index = index;
@ -3526,26 +3536,24 @@ static PyObject *
listreviter_next(PyObject *self)
{
listreviterobject *it = (listreviterobject *)self;
PyObject *item;
Py_ssize_t index;
PyListObject *seq;
assert(it != NULL);
seq = it->it_seq;
if (seq == NULL) {
return NULL;
}
PyListObject *seq = it->it_seq;
assert(PyList_Check(seq));
index = it->it_index;
if (index>=0 && index < PyList_GET_SIZE(seq)) {
item = PyList_GET_ITEM(seq, index);
it->it_index--;
return Py_NewRef(item);
Py_ssize_t index = LOAD_SSIZE(it->it_index);
if (index < 0) {
return NULL;
}
it->it_index = -1;
PyObject *item = list_get_item_ref(seq, index);
if (item != NULL) {
STORE_SSIZE(it->it_index, index - 1);
return item;
}
STORE_SSIZE(it->it_index, -1);
#ifndef Py_GIL_DISABLED
it->it_seq = NULL;
Py_DECREF(seq);
#endif
return NULL;
}
@ -3553,7 +3561,8 @@ static PyObject *
listreviter_len(PyObject *self, PyObject *Py_UNUSED(ignored))
{
listreviterobject *it = (listreviterobject *)self;
Py_ssize_t len = it->it_index + 1;
Py_ssize_t index = LOAD_SSIZE(it->it_index);
Py_ssize_t len = index + 1;
if (it->it_seq == NULL || PyList_GET_SIZE(it->it_seq) < len)
len = 0;
return PyLong_FromSsize_t(len);
@ -3588,6 +3597,7 @@ static PyObject *
listiter_reduce_general(void *_it, int forward)
{
PyObject *list;
PyObject *iter;
/* _PyEval_GetBuiltin can invoke arbitrary code,
* call must be before access of iterator pointers.
@ -3595,29 +3605,21 @@ listiter_reduce_general(void *_it, int forward)
/* the objects are not the same, index is of different types! */
if (forward) {
PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter));
if (!iter) {
return NULL;
}
iter = _PyEval_GetBuiltin(&_Py_ID(iter));
_PyListIterObject *it = (_PyListIterObject *)_it;
if (it->it_seq) {
if (it->it_index >= 0) {
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
}
Py_DECREF(iter);
} else {
PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed));
if (!reversed) {
return NULL;
}
iter = _PyEval_GetBuiltin(&_Py_ID(reversed));
listreviterobject *it = (listreviterobject *)_it;
if (it->it_seq) {
return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index);
if (it->it_index >= 0) {
return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index);
}
Py_DECREF(reversed);
}
/* empty iterator, create an empty list */
list = PyList_New(0);
if (list == NULL)
return NULL;
return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), list);
return Py_BuildValue("N(N)", iter, list);
}

8
Python/bytecodes.c
View File

@ -2606,11 +2606,14 @@ dummy_func(
assert(Py_TYPE(iter) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) {
if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
@ -2624,8 +2627,7 @@ dummy_func(
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
DEOPT_IF(seq == NULL);
DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq));
DEOPT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq));
}
op(_ITER_NEXT_LIST, (iter -- iter, next)) {

3
Python/executor_cases.c.h generated
View File

@ -2201,8 +2201,7 @@
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
if (seq == NULL) goto deoptimize;
if (it->it_index >= PyList_GET_SIZE(seq)) goto deoptimize;
if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize;
break;
}

5
Python/generated_cases.c.h generated
View File

@ -2541,11 +2541,14 @@
assert(Py_TYPE(iter) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) {
if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */