bpo-29942: Fix the use of recursion in itertools.chain.from_iterable. (#889)

Fix the use of recursion in itertools.chain.from_iterable. Using recursion
is unnecessary, and can easily cause stack overflows, especially when
building in low optimization modes or with Py_DEBUG enabled.
This commit is contained in:
T. Wouters 2017-03-30 09:58:35 -07:00 committed by GitHub
parent 06e522521c
commit 5466d4af5f
3 changed files with 39 additions and 24 deletions

View File

@ -1976,6 +1976,14 @@ class RegressionTests(unittest.TestCase):
self.assertRaises(AssertionError, list, cycle(gen1()))
self.assertEqual(hist, [0,1])
def test_long_chain_of_empty_iterables(self):
# Make sure itertools.chain doesn't run into recursion limits when
# dealing with long chains of empty iterables. Even with a high
# number this would probably only fail in Py_DEBUG mode.
it = chain.from_iterable(() for unused in range(10000000))
with self.assertRaises(StopIteration):
next(it)
class SubclassWithKwargsTest(unittest.TestCase):
def test_keywords_in_subclass(self):
# count is not subclassable...

View File

@ -301,6 +301,9 @@ Extension Modules
Library
-------
- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering
long runs of empty iterables.
- bpo-10030: Sped up reading encrypted ZIP files by 2 times.
- bpo-29204: Element.getiterator() and the html parameter of XMLParser() were

View File

@ -1864,33 +1864,37 @@ chain_next(chainobject *lz)
{
PyObject *item;
if (lz->source == NULL)
return NULL; /* already stopped */
if (lz->active == NULL) {
PyObject *iterable = PyIter_Next(lz->source);
if (iterable == NULL) {
Py_CLEAR(lz->source);
return NULL; /* no more input sources */
}
lz->active = PyObject_GetIter(iterable);
Py_DECREF(iterable);
/* lz->source is the iterator of iterables. If it's NULL, we've already
* consumed them all. lz->active is the current iterator. If it's NULL,
* we should grab a new one from lz->source. */
while (lz->source != NULL) {
if (lz->active == NULL) {
Py_CLEAR(lz->source);
return NULL; /* input not iterable */
PyObject *iterable = PyIter_Next(lz->source);
if (iterable == NULL) {
Py_CLEAR(lz->source);
return NULL; /* no more input sources */
}
lz->active = PyObject_GetIter(iterable);
Py_DECREF(iterable);
if (lz->active == NULL) {
Py_CLEAR(lz->source);
return NULL; /* input not iterable */
}
}
item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active);
if (item != NULL)
return item;
if (PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_StopIteration))
PyErr_Clear();
else
return NULL; /* input raised an exception */
}
/* lz->active is consumed, try with the next iterable. */
Py_CLEAR(lz->active);
}
item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active);
if (item != NULL)
return item;
if (PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_StopIteration))
PyErr_Clear();
else
return NULL; /* input raised an exception */
}
Py_CLEAR(lz->active);
return chain_next(lz); /* recurse and use next active */
/* Everything had been consumed already. */
return NULL;
}
static PyObject *