Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-123884 Tee of tee was not producing n independent iterators (gh-124490)

This commit is contained in:
Raymond Hettinger 2024-09-25 13:38:05 -07:00 committed by GitHub
parent fb6bd31cb7
commit 909c6f7189
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 89 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Doc/library/itertools.rst
View File

@ -691,25 +691,36 @@ loops that truncate the stream.
def tee(iterable, n=2):
if n < 0:
raise ValueError('n must be >= 0')
iterator = iter(iterable)
shared_link = [None, None]
return tuple(_tee(iterator, shared_link) for _ in range(n))
raise ValueError
if n == 0:
return ()
iterator = _tee(iterable)
result = [iterator]
for _ in range(n - 1):
result.append(_tee(iterator))
return tuple(result)
def _tee(iterator, link):
try:
while True:
if link[1] is None:
link[0] = next(iterator)
link[1] = [None, None]
value, link = link
yield value
except StopIteration:
return
class _tee:
Once a :func:`tee` has been created, the original *iterable* should not be
used anywhere else; otherwise, the *iterable* could get advanced without
the tee objects being informed.
def __init__(self, iterable):
it = iter(iterable)
if isinstance(it, _tee):
self.iterator = it.iterator
self.link = it.link
else:
self.iterator = it
self.link = [None, None]
def __iter__(self):
return self
def __next__(self):
link = self.link
if link[1] is None:
link[0] = next(self.iterator)
link[1] = [None, None]
value, self.link = link
return value
When the input *iterable* is already a tee iterator object, all
members of the return tuple are constructed as if they had been

1
Include/internal/pycore_global_objects_fini_generated.h generated
View File

@ -604,7 +604,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__classdictcell__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__complex__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__contains__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__copy__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ctypes_from_outparam__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__del__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__delattr__));

1
Include/internal/pycore_global_strings.h
View File

@ -93,7 +93,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__classdictcell__)
STRUCT_FOR_ID(__complex__)
STRUCT_FOR_ID(__contains__)
STRUCT_FOR_ID(__copy__)
STRUCT_FOR_ID(__ctypes_from_outparam__)
STRUCT_FOR_ID(__del__)
STRUCT_FOR_ID(__delattr__)

1
Include/internal/pycore_runtime_init_generated.h generated
View File

@ -602,7 +602,6 @@ extern "C" {
INIT_ID(__classdictcell__), \
INIT_ID(__complex__), \
INIT_ID(__contains__), \
INIT_ID(__copy__), \
INIT_ID(__ctypes_from_outparam__), \
INIT_ID(__del__), \
INIT_ID(__delattr__), \

4
Include/internal/pycore_unicodeobject_generated.h generated
View File

@ -172,10 +172,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(__copy__);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(__ctypes_from_outparam__);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));

84
Lib/test/test_itertools.py
View File

@ -1249,10 +1249,11 @@ class TestBasicOps(unittest.TestCase):
self.assertEqual(len(result), n)
self.assertEqual([list(x) for x in result], [list('abc')]*n)
# tee pass-through to copyable iterator
# tee objects are independent (see bug gh-123884)
a, b = tee('abc')
c, d = tee(a)
self.assertTrue(a is c)
e, f = tee(c)
self.assertTrue(len({a, b, c, d, e, f}) == 6)
# test tee_new
t1, t2 = tee('abc')
@ -1759,21 +1760,36 @@ class TestPurePythonRoughEquivalents(unittest.TestCase):
def tee(iterable, n=2):
if n < 0:
raise ValueError('n must be >= 0')
iterator = iter(iterable)
shared_link = [None, None]
return tuple(_tee(iterator, shared_link) for _ in range(n))
raise ValueError
if n == 0:
return ()
iterator = _tee(iterable)
result = [iterator]
for _ in range(n - 1):
result.append(_tee(iterator))
return tuple(result)
def _tee(iterator, link):
try:
while True:
if link[1] is None:
link[0] = next(iterator)
link[1] = [None, None]
value, link = link
yield value
except StopIteration:
return
class _tee:
def __init__(self, iterable):
it = iter(iterable)
if isinstance(it, _tee):
self.iterator = it.iterator
self.link = it.link
else:
self.iterator = it
self.link = [None, None]
def __iter__(self):
return self
def __next__(self):
link = self.link
if link[1] is None:
link[0] = next(self.iterator)
link[1] = [None, None]
value, self.link = link
return value
# End tee() recipe #############################################
@ -1819,12 +1835,10 @@ class TestPurePythonRoughEquivalents(unittest.TestCase):
self.assertRaises(TypeError, tee, [1,2], 'x')
self.assertRaises(TypeError, tee, [1,2], 3, 'x')
# Tests not applicable to the tee() recipe
if False:
# tee object should be instantiable
a, b = tee('abc')
c = type(a)('def')
self.assertEqual(list(c), list('def'))
# tee object should be instantiable
a, b = tee('abc')
c = type(a)('def')
self.assertEqual(list(c), list('def'))
# test long-lagged and multi-way split
a, b, c = tee(range(2000), 3)
@ -1845,21 +1859,19 @@ class TestPurePythonRoughEquivalents(unittest.TestCase):
self.assertEqual(len(result), n)
self.assertEqual([list(x) for x in result], [list('abc')]*n)
# tee objects are independent (see bug gh-123884)
a, b = tee('abc')
c, d = tee(a)
e, f = tee(c)
self.assertTrue(len({a, b, c, d, e, f}) == 6)
# Tests not applicable to the tee() recipe
if False:
# tee pass-through to copyable iterator
a, b = tee('abc')
c, d = tee(a)
self.assertTrue(a is c)
# test tee_new
t1, t2 = tee('abc')
tnew = type(t1)
self.assertRaises(TypeError, tnew)
self.assertRaises(TypeError, tnew, 10)
t3 = tnew(t1)
self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc'))
# test tee_new
t1, t2 = tee('abc')
tnew = type(t1)
self.assertRaises(TypeError, tnew)
self.assertRaises(TypeError, tnew, 10)
t3 = tnew(t1)
self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc'))
# test that tee objects are weak referencable
a, b = tee(range(10))

4
Misc/NEWS.d/next/Library/2024-09-24-22-38-51.gh-issue-123884.iEPTK4.rst Normal file
View File

@ -0,0 +1,4 @@
Fixed bug in itertools.tee() handling of other tee inputs (a tee in a tee).
The output now has the promised *n* independent new iterators. Formerly,
the first iterator was identical (not independent) to the input iterator.
This would sometimes give surprising results.

36
Modules/itertoolsmodule.c
View File

@ -1036,7 +1036,7 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n)
/*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/
{
Py_ssize_t i;
PyObject *it, *copyable, *copyfunc, *result;
PyObject *it, *to, *result;
if (n < 0) {
PyErr_SetString(PyExc_ValueError, "n must be >= 0");
@ -1053,41 +1053,23 @@ itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n)
return NULL;
}
if (PyObject_GetOptionalAttr(it, &_Py_ID(__copy__), &copyfunc) < 0) {
Py_DECREF(it);
itertools_state *state = get_module_state(module);
to = tee_fromiterable(state, it);
Py_DECREF(it);
if (to == NULL) {
Py_DECREF(result);
return NULL;
}
if (copyfunc != NULL) {
copyable = it;
}
else {
itertools_state *state = get_module_state(module);
copyable = tee_fromiterable(state, it);
Py_DECREF(it);
if (copyable == NULL) {
Py_DECREF(result);
return NULL;
}
copyfunc = PyObject_GetAttr(copyable, &_Py_ID(__copy__));
if (copyfunc == NULL) {
Py_DECREF(copyable);
Py_DECREF(result);
return NULL;
}
}
PyTuple_SET_ITEM(result, 0, copyable);
PyTuple_SET_ITEM(result, 0, to);
for (i = 1; i < n; i++) {
copyable = _PyObject_CallNoArgs(copyfunc);
if (copyable == NULL) {
Py_DECREF(copyfunc);
to = tee_copy((teeobject *)to, NULL);
if (to == NULL) {
Py_DECREF(result);
return NULL;
}
PyTuple_SET_ITEM(result, i, copyable);
PyTuple_SET_ITEM(result, i, to);
}
Py_DECREF(copyfunc);
return result;
}