From 81dd8b9594d88ff1d2c8f5efea687645bbc36d6f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 14 Nov 2009 18:09:17 +0000 Subject: [PATCH] use some more itertools magic to make '' be yielded after readline is done --- Lib/tokenize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index b8ee2c854f9..f82922b7f08 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -379,10 +379,11 @@ def tokenize(readline): """ # This import is here to avoid problems when the itertools module is not # built yet and tokenize is imported. - from itertools import chain + from itertools import chain, repeat encoding, consumed = detect_encoding(readline) - rl_iter = iter(readline, "") - return _tokenize(chain(consumed, rl_iter).__next__, encoding) + rl_gen = iter(readline, b"") + empty = repeat(b"") + return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding) def _tokenize(readline, encoding):