Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

use some more itertools magic to make '' be yielded after readline is done

This commit is contained in:
Benjamin Peterson 2009-11-14 18:09:17 +00:00
parent 21db77e396
commit 81dd8b9594
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Lib/tokenize.py
View File

@ -379,10 +379,11 @@ def tokenize(readline):
"""
# This import is here to avoid problems when the itertools module is not
# built yet and tokenize is imported.
from itertools import chain
from itertools import chain, repeat
encoding, consumed = detect_encoding(readline)
rl_iter = iter(readline, "")
return _tokenize(chain(consumed, rl_iter).__next__, encoding)
rl_gen = iter(readline, b"")
empty = repeat(b"")
return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
def _tokenize(readline, encoding):