From 25dded041fe532fcb041b6e68582bf76b4968132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Wed, 5 Mar 2014 13:47:57 +0000 Subject: [PATCH] Make the various iterators' "setstate" sliently and consistently clip the index. This avoids the possibility of setting an iterator to an invalid state. --- Lib/test/test_range.py | 12 ++++++++++++ Modules/arraymodule.c | 2 ++ Objects/bytearrayobject.c | 10 +++++++--- Objects/bytesobject.c | 10 +++++++--- Objects/listobject.c | 2 ++ Objects/rangeobject.c | 31 +++++++++++++++++++++++++++---- Objects/tupleobject.c | 4 ++-- Objects/unicodeobject.c | 10 +++++++--- 8 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 063f320bfe6..aab98abf4da 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -380,6 +380,18 @@ class RangeTest(unittest.TestCase): self.assertEqual(list(it), data[1:]) def test_exhausted_iterator_pickling(self): + r = range(2**65, 2**65+2) + i = iter(r) + while True: + r = next(i) + if r == 2**65+1: + break + d = pickle.dumps(i) + i2 = pickle.loads(d) + self.assertEqual(list(i), []) + self.assertEqual(list(i2), []) + + def test_large_exhausted_iterator_pickling(self): r = range(20) i = iter(r) while True: diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 96c9e5bba75..3aba9adbeb2 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2798,6 +2798,8 @@ arrayiter_setstate(arrayiterobject *it, PyObject *state) return NULL; if (index < 0) index = 0; + else if (index > Py_SIZE(it->ao)) + index = Py_SIZE(it->ao); /* iterator exhausted */ it->index = index; Py_RETURN_NONE; } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 60b281179d9..50667a646a3 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -3043,9 +3043,13 @@ bytearrayiter_setstate(bytesiterobject *it, PyObject *state) Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (index < 0) - index = 0; - it->it_index = index; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + else if (index > PyByteArray_GET_SIZE(it->it_seq)) + index = PyByteArray_GET_SIZE(it->it_seq); /* iterator exhausted */ + it->it_index = index; + } Py_RETURN_NONE; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9dcb74e8fdb..f6d16dafdb5 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2996,9 +2996,13 @@ striter_setstate(striterobject *it, PyObject *state) Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (index < 0) - index = 0; - it->it_index = index; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + else if (index > PyBytes_GET_SIZE(it->it_seq)) + index = PyBytes_GET_SIZE(it->it_seq); /* iterator exhausted */ + it->it_index = index; + } Py_RETURN_NONE; } diff --git a/Objects/listobject.c b/Objects/listobject.c index 6e0d094154e..143c7b37883 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2803,6 +2803,8 @@ listiter_setstate(listiterobject *it, PyObject *state) if (it->it_seq != NULL) { if (index < 0) index = 0; + else if (index > PyList_GET_SIZE(it->it_seq)) + index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */ it->it_index = index; } Py_RETURN_NONE; diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 4b218b89617..f2030c3cd2a 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1000,10 +1000,11 @@ rangeiter_setstate(rangeiterobject *r, PyObject *state) long index = PyLong_AsLong(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (index < 0 || index > r->len) { - PyErr_SetString(PyExc_ValueError, "index out of range"); - return NULL; - } + /* silently clip the index value */ + if (index < 0) + index = 0; + else if (index > r->len) + index = r->len; /* exhausted iterator */ r->index = index; Py_RETURN_NONE; } @@ -1178,6 +1179,28 @@ longrangeiter_reduce(longrangeiterobject *r) static PyObject * longrangeiter_setstate(longrangeiterobject *r, PyObject *state) { + int cmp; + + /* clip the value */ + PyObject *zero = PyLong_FromLong(0); + if (zero == NULL) + return NULL; + cmp = PyObject_RichCompareBool(state, zero, Py_LT); + if (cmp > 0) { + Py_CLEAR(r->index); + r->index = zero; + Py_RETURN_NONE; + } + Py_DECREF(zero); + if (cmp < 0) + return NULL; + + cmp = PyObject_RichCompareBool(r->len, state, Py_LT); + if (cmp < 0) + return NULL; + if (cmp > 0) + state = r->len; + Py_CLEAR(r->index); r->index = state; Py_INCREF(r->index); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 123df8cef3c..f81559504fc 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -998,8 +998,8 @@ tupleiter_setstate(tupleiterobject *it, PyObject *state) if (it->it_seq != NULL) { if (index < 0) index = 0; - else if (it->it_seq != NULL && index > PyTuple_GET_SIZE(it->it_seq)) - index = PyTuple_GET_SIZE(it->it_seq); + else if (index > PyTuple_GET_SIZE(it->it_seq)) + index = PyTuple_GET_SIZE(it->it_seq); /* exhausted iterator */ it->it_index = index; } Py_RETURN_NONE; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 03007531bd8..4085d22b716 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14420,9 +14420,13 @@ unicodeiter_setstate(unicodeiterobject *it, PyObject *state) Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (index < 0) - index = 0; - it->it_index = index; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + else if (index > PyUnicode_GET_LENGTH(it->it_seq)) + index = PyUnicode_GET_LENGTH(it->it_seq); /* iterator truncated */ + it->it_index = index; + } Py_RETURN_NONE; }