Issue #26494: Fixed crash on iterating exhausting iterators.

Affected classes are generic sequence iterators, iterators of str, bytes,
bytearray, list, tuple, set, frozenset, dict, OrderedDict, corresponding
views and os.scandir() iterator.
This commit is contained in:
Serhiy Storchaka 2016-03-30 20:40:02 +03:00
parent 13b3acd13e
commit fbb1c5ee06
19 changed files with 94 additions and 24 deletions

View File

@ -5,6 +5,7 @@ Tests common to tuple, list and UserList.UserList
import unittest
import sys
import pickle
from test import support
# Various iterables
# This is used for checking the constructor (here and in test_deque.py)
@ -408,3 +409,7 @@ class CommonTest(unittest.TestCase):
lst2 = pickle.loads(pickle.dumps(lst, proto))
self.assertEqual(lst2, lst)
self.assertNotEqual(id(lst2), id(lst))
def test_free_after_iterating(self):
support.check_free_after_iterating(self, iter, self.type2test)
support.check_free_after_iterating(self, reversed, self.type2test)

View File

@ -2366,3 +2366,22 @@ def run_in_subinterp(code):
"memory allocations")
import _testcapi
return _testcapi.run_in_subinterp(code)
def check_free_after_iterating(test, iter, cls, args=()):
class A(cls):
def __del__(self):
nonlocal done
done = True
try:
next(it)
except StopIteration:
pass
done = False
it = iter(A(*args))
# Issue 26494: Shouldn't crash
test.assertRaises(StopIteration, next, it)
# The sequence should be deallocated just after the end of iterating
gc_collect()
test.assertTrue(done)

View File

@ -747,6 +747,10 @@ class BaseBytesTest:
self.assertRaisesRegex(TypeError, r'\bendswith\b', b.endswith,
x, None, None, None)
def test_free_after_iterating(self):
test.support.check_free_after_iterating(self, iter, self.type2test)
test.support.check_free_after_iterating(self, reversed, self.type2test)
class BytesTest(BaseBytesTest, unittest.TestCase):
type2test = bytes

View File

@ -905,6 +905,10 @@ class TestSequence(seq_tests.CommonTest):
# For now, bypass tests that require slicing
pass
def test_free_after_iterating(self):
# For now, bypass tests that require slicing
self.skipTest("Exhausted deque iterator doesn't free a deque")
#==============================================================================
libreftest = """

View File

@ -952,6 +952,12 @@ class DictTest(unittest.TestCase):
d = {X(): 0, 1: 1}
self.assertRaises(RuntimeError, d.update, other)
def test_free_after_iterating(self):
support.check_free_after_iterating(self, iter, dict)
support.check_free_after_iterating(self, lambda d: iter(d.keys()), dict)
support.check_free_after_iterating(self, lambda d: iter(d.values()), dict)
support.check_free_after_iterating(self, lambda d: iter(d.items()), dict)
from test import mapping_tests
class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol):

View File

@ -3,6 +3,7 @@
import sys
import unittest
from test.support import run_unittest, TESTFN, unlink, cpython_only
from test.support import check_free_after_iterating
import pickle
import collections.abc
@ -980,6 +981,9 @@ class TestCase(unittest.TestCase):
self.assertEqual(next(it), 0)
self.assertEqual(next(it), 1)
def test_free_after_iterating(self):
check_free_after_iterating(self, iter, SequenceClass, (0,))
def test_main():
run_unittest(TestCase)

View File

@ -598,6 +598,12 @@ class OrderedDictTests:
gc.collect()
self.assertIsNone(r())
def test_free_after_iterating(self):
support.check_free_after_iterating(self, iter, self.OrderedDict)
support.check_free_after_iterating(self, lambda d: iter(d.keys()), self.OrderedDict)
support.check_free_after_iterating(self, lambda d: iter(d.values()), self.OrderedDict)
support.check_free_after_iterating(self, lambda d: iter(d.items()), self.OrderedDict)
class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase):

View File

@ -362,6 +362,9 @@ class TestJointOps:
gc.collect()
self.assertTrue(ref() is None, "Cycle was not collected")
def test_free_after_iterating(self):
support.check_free_after_iterating(self, iter, self.thetype)
class TestSet(TestJointOps, unittest.TestCase):
thetype = set
basetype = set

View File

@ -2729,6 +2729,10 @@ class UnicodeTest(string_tests.CommonTest,
# Check that the second call returns the same result
self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
def test_free_after_iterating(self):
support.check_free_after_iterating(self, iter, str)
support.check_free_after_iterating(self, reversed, str)
class StringModuleTest(unittest.TestCase):
def test_formatter_parser(self):

View File

@ -10,6 +10,11 @@ Release date: tba
Core and Builtins
-----------------
- Issue #26494: Fixed crash on iterating exhausting iterators.
Affected classes are generic sequence iterators, iterators of str, bytes,
bytearray, list, tuple, set, frozenset, dict, OrderedDict, corresponding
views and os.scandir() iterator.
- Issue #26581: If coding cookie is specified multiple times on a line in
Python source code file, only the first one is taken to account.

View File

@ -11928,13 +11928,15 @@ typedef struct {
static void
ScandirIterator_close(ScandirIterator *iterator)
{
if (iterator->handle == INVALID_HANDLE_VALUE)
HANDLE handle = iterator->handle;
if (handle == INVALID_HANDLE_VALUE)
return;
Py_BEGIN_ALLOW_THREADS
FindClose(iterator->handle);
Py_END_ALLOW_THREADS
iterator->handle = INVALID_HANDLE_VALUE;
Py_BEGIN_ALLOW_THREADS
FindClose(handle);
Py_END_ALLOW_THREADS
}
static PyObject *
@ -11984,13 +11986,15 @@ ScandirIterator_iternext(ScandirIterator *iterator)
static void
ScandirIterator_close(ScandirIterator *iterator)
{
if (!iterator->dirp)
DIR *dirp = iterator->dirp;
if (!dirp)
return;
Py_BEGIN_ALLOW_THREADS
closedir(iterator->dirp);
Py_END_ALLOW_THREADS
iterator->dirp = NULL;
Py_BEGIN_ALLOW_THREADS
closedir(dirp);
Py_END_ALLOW_THREADS
return;
}

View File

@ -3186,8 +3186,8 @@ bytearrayiter_next(bytesiterobject *it)
return item;
}
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}

View File

@ -3628,8 +3628,8 @@ striter_next(striterobject *it)
return item;
}
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}

View File

@ -2985,8 +2985,8 @@ static PyObject *dictiter_iternextkey(dictiterobject *di)
return key;
fail:
Py_DECREF(d);
di->di_dict = NULL;
Py_DECREF(d);
return NULL;
}
@ -3066,8 +3066,8 @@ static PyObject *dictiter_iternextvalue(dictiterobject *di)
return value;
fail:
Py_DECREF(d);
di->di_dict = NULL;
Py_DECREF(d);
return NULL;
}
@ -3161,8 +3161,8 @@ static PyObject *dictiter_iternextitem(dictiterobject *di)
return result;
fail:
Py_DECREF(d);
di->di_dict = NULL;
Py_DECREF(d);
return NULL;
}

View File

@ -69,8 +69,8 @@ iter_iternext(PyObject *iterator)
PyErr_ExceptionMatches(PyExc_StopIteration))
{
PyErr_Clear();
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
}
return NULL;
}

View File

@ -2782,8 +2782,8 @@ listiter_next(listiterobject *it)
return item;
}
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}
@ -2912,9 +2912,17 @@ static PyObject *
listreviter_next(listreviterobject *it)
{
PyObject *item;
Py_ssize_t index = it->it_index;
PyListObject *seq = it->it_seq;
Py_ssize_t index;
PyListObject *seq;
assert(it != NULL);
seq = it->it_seq;
if (seq == NULL) {
return NULL;
}
assert(PyList_Check(seq));
index = it->it_index;
if (index>=0 && index < PyList_GET_SIZE(seq)) {
item = PyList_GET_ITEM(seq, index);
it->it_index--;
@ -2922,10 +2930,8 @@ listreviter_next(listreviterobject *it)
return item;
}
it->it_index = -1;
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}

View File

@ -839,8 +839,8 @@ static PyObject *setiter_iternext(setiterobject *si)
return key;
fail:
Py_DECREF(so);
si->si_set = NULL;
Py_DECREF(so);
return NULL;
}

View File

@ -964,8 +964,8 @@ tupleiter_next(tupleiterobject *it)
return item;
}
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}

View File

@ -15149,8 +15149,8 @@ unicodeiter_next(unicodeiterobject *it)
return item;
}
Py_DECREF(seq);
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}