Issue #26494: Fixed crash on iterating exhausting iterators.

Affected classes are generic sequence iterators, iterators of bytearray,
list, tuple, set, frozenset, dict, OrderedDict and corresponding views.

Lib/test/seq_tests.py

@@ -4,6 +4,7 @@ Tests common to tuple, list and UserList.UserList
 
 import unittest
 import sys
+from test import test_support as support
 
 # Various iterables
 # This is used for checking the constructor (here and in test_deque.py)
@@ -402,3 +403,7 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(a.index(0, -4*sys.maxint, 4*sys.maxint), 2)
         self.assertRaises(ValueError, a.index, 0, 4*sys.maxint,-4*sys.maxint)
         self.assertRaises(ValueError, a.index, 2, 0, -10)
+
+    def test_free_after_iterating(self):
+        support.check_free_after_iterating(self, iter, self.type2test)
+        support.check_free_after_iterating(self, reversed, self.type2test)

Lib/test/test_bytes.py

@@ -518,6 +518,10 @@ class BaseBytesTest(unittest.TestCase):
         self.assertRaisesRegexp(TypeError, r'\bendswith\b', b.endswith,
                                 x, None, None, None)
 
+    def test_free_after_iterating(self):
+        test.test_support.check_free_after_iterating(self, iter, self.type2test)
+        test.test_support.check_free_after_iterating(self, reversed, self.type2test)
+
 
 class ByteArrayTest(BaseBytesTest):
     type2test = bytearray

Lib/test/test_deque.py

@@ -669,6 +669,10 @@ class TestSubclassWithKwargs(unittest.TestCase):
         # SF bug #1486663 -- this used to erroneously raise a TypeError
         SubclassWithKwargs(newarg=1)
 
+    def test_free_after_iterating(self):
+        # For now, bypass tests that require slicing
+        self.skipTest("Exhausted deque iterator doesn't free a deque")
+
 #==============================================================================
 
 libreftest = """

Lib/test/test_dict.py

@@ -681,6 +681,15 @@ class DictTest(unittest.TestCase):
         self._tracked(MyDict())
 
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, dict)
+        test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), dict)
+        test_support.check_free_after_iterating(self, lambda d: d.itervalues(), dict)
+        test_support.check_free_after_iterating(self, lambda d: d.iteritems(), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), dict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), dict)
+
 from test import mapping_tests
 
 class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol):

Lib/test/test_iter.py

@@ -2,7 +2,8 @@
 
 import unittest
 from test.test_support import run_unittest, TESTFN, unlink, have_unicode, \
-                              check_py3k_warnings, cpython_only
+                              check_py3k_warnings, cpython_only, \
+                              check_free_after_iterating
 
 # Test result of triple loop (too big to inline)
 TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -921,6 +922,9 @@ class TestCase(unittest.TestCase):
         lst.extend(gen())
         self.assertEqual(len(lst), 760)
 
+    def test_free_after_iterating(self):
+        check_free_after_iterating(self, iter, SequenceClass, (0,))
+
 
 def test_main():
     run_unittest(TestCase)

Lib/test/test_ordered_dict.py

@@ -267,6 +267,15 @@ class TestOrderedDict(unittest.TestCase):
         items = [('a', 1), ('c', 3), ('b', 2)]
         self.assertEqual(list(MyOD(items).items()), items)
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.itervalues(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: d.iteritems(), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), OrderedDict)
+        test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), OrderedDict)
+
 class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
     type2test = OrderedDict
 

Lib/test/test_set.py

@@ -340,6 +340,9 @@ class TestJointOps(unittest.TestCase):
         gc.collect()
         self.assertTrue(ref() is None, "Cycle was not collected")
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, self.thetype)
+
 class TestSet(TestJointOps):
     thetype = set
 

Lib/test/test_support.py

@@ -1659,3 +1659,21 @@ def strip_python_stderr(stderr):
     """
     stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip()
     return stderr
+
+
+def check_free_after_iterating(test, iter, cls, args=()):
+    class A(cls):
+        def __del__(self):
+            done[0] = True
+            try:
+                next(it)
+            except StopIteration:
+                pass
+
+    done = [False]
+    it = iter(A(*args))
+    # Issue 26494: Shouldn't crash
+    test.assertRaises(StopIteration, next, it)
+    # The sequence should be deallocated just after the end of iterating
+    gc_collect()
+    test.assertTrue(done[0])

Lib/test/test_unicode.py

@@ -1857,6 +1857,11 @@ class UnicodeTest(
                     unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"),
                     '123' + exp)
 
+    def test_free_after_iterating(self):
+        test_support.check_free_after_iterating(self, iter, unicode)
+        test_support.check_free_after_iterating(self, reversed, unicode)
+
+
 def test_main():
     test_support.run_unittest(__name__)
 

Misc/NEWS

@@ -10,6 +10,10 @@ What's New in Python 2.7.12?
 Core and Builtins
 -----------------
 
+- Issue #26494: Fixed crash on iterating exhausting iterators.
+  Affected classes are generic sequence iterators, iterators of bytearray,
+  list, tuple, set, frozenset, dict, OrderedDict and corresponding views.
+
 - Issue #26581: If coding cookie is specified multiple times on a line in
   Python source code file, only the first one is taken to account.
 

Objects/bytearrayobject.c

@@ -2982,8 +2982,8 @@ bytearrayiter_next(bytesiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 

Objects/dictobject.c

@@ -2586,8 +2586,8 @@ static PyObject *dictiter_iternextkey(dictiterobject *di)
     return key;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 
@@ -2658,8 +2658,8 @@ static PyObject *dictiter_iternextvalue(dictiterobject *di)
     return value;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 
@@ -2744,8 +2744,8 @@ static PyObject *dictiter_iternextitem(dictiterobject *di)
     return result;
 
 fail:
-    Py_DECREF(d);
     di->di_dict = NULL;
+    Py_DECREF(d);
     return NULL;
 }
 

Objects/iterobject.c

@@ -69,8 +69,8 @@ iter_iternext(PyObject *iterator)
         PyErr_ExceptionMatches(PyExc_StopIteration))
     {
         PyErr_Clear();
-        Py_DECREF(seq);
         it->it_seq = NULL;
+        Py_DECREF(seq);
     }
     return NULL;
 }

Objects/listobject.c

@@ -2915,8 +2915,8 @@ listiter_next(listiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 
@@ -3018,9 +3018,17 @@ static PyObject *
 listreviter_next(listreviterobject *it)
 {
     PyObject *item;
-    Py_ssize_t index = it->it_index;
-    PyListObject *seq = it->it_seq;
+    Py_ssize_t index;
+    PyListObject *seq;
 
+    assert(it != NULL);
+    seq = it->it_seq;
+    if (seq == NULL) {
+        return NULL;
+    }
+    assert(PyList_Check(seq));
+
+    index = it->it_index;
     if (index>=0 && index < PyList_GET_SIZE(seq)) {
         item = PyList_GET_ITEM(seq, index);
         it->it_index--;
@@ -3028,10 +3036,8 @@ listreviter_next(listreviterobject *it)
         return item;
     }
     it->it_index = -1;
-    if (seq != NULL) {
-        it->it_seq = NULL;
-        Py_DECREF(seq);
-    }
+    it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }
 

Objects/setobject.c

@@ -871,8 +871,8 @@ static PyObject *setiter_iternext(setiterobject *si)
     return key;
 
 fail:
-    Py_DECREF(so);
     si->si_set = NULL;
+    Py_DECREF(so);
     return NULL;
 }
 

Objects/tupleobject.c

@@ -966,8 +966,8 @@ tupleiter_next(tupleiterobject *it)
         return item;
     }
 
-    Py_DECREF(seq);
     it->it_seq = NULL;
+    Py_DECREF(seq);
     return NULL;
 }