diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index a4963f8c634..bbac59bd6c8 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -627,9 +627,17 @@ class UntokenizeTest(TestCase): 'start (1,3) precedes previous end (2,2)') self.assertRaises(ValueError, u.add_whitespace, (2,1)) -__test__ = {"doctests" : doctests, 'decistmt': decistmt} + def test_iter_compat(self): + u = Untokenizer() + token = (NAME, 'Hello') + u.compat(token, iter([])) + self.assertEqual(u.tokens, ["Hello "]) + u = Untokenizer() + self.assertEqual(u.untokenize(iter([token])), 'Hello ') +__test__ = {"doctests" : doctests, 'decistmt': decistmt} + def test_main(): from test import test_tokenize test_support.run_doctest(test_tokenize, True) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index a5bfa5adf51..54441ce8677 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -26,6 +26,7 @@ __author__ = 'Ka-Ping Yee ' __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' 'Skip Montanaro, Raymond Hettinger') +from itertools import chain import string, re from token import * @@ -192,9 +193,10 @@ class Untokenizer: self.tokens.append(" " * col_offset) def untokenize(self, iterable): - for t in iterable: + it = iter(iterable) + for t in it: if len(t) == 2: - self.compat(t, iterable) + self.compat(t, it) break tok_type, token, start, end, line = t self.add_whitespace(start) @@ -206,16 +208,12 @@ class Untokenizer: return "".join(self.tokens) def compat(self, token, iterable): - startline = False indents = [] toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True + startline = token[0] in (NEWLINE, NL) prevstring = False - for tok in iterable: + + for tok in chain([token], iterable): toknum, tokval = tok[:2] if toknum in (NAME, NUMBER): diff --git a/Misc/NEWS b/Misc/NEWS index ae5cac34260..dd00110edf2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -42,6 +42,9 @@ Library - Issue #17671: Fixed a crash when use non-initialized io.BufferedRWPair. Based on patch by Stephen Tu. +- Issue #8478: Untokenizer.compat processes first token from iterator input. + Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees. + - Issue #20594: Avoid name clash with the libc function posix_close. - Issue #19856: shutil.move() failed to move a directory to other directory