diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index a51e7818670..489f68fade9 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -550,6 +550,10 @@ Evil tabs NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) + +Pathological whitespace (http://bugs.python.org/issue16152) + >>> dump_tokens("@ ") + OP '@' (1, 0) (1, 1) """ diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1cba6e5d9e3..ca7b07493cf 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -95,7 +95,7 @@ ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) tokenprog, pseudoprog, single3prog, double3prog = map( @@ -362,6 +362,8 @@ def generate_tokens(readline): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if initial in numchars or \ diff --git a/Misc/ACKS b/Misc/ACKS index 129a116d8db..01bc45ecb81 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -64,6 +64,7 @@ Des Barry Ulf Bartelt Don Bashford Nick Bastin +Ned Batchelder Jeff Bauer Mike Bayer Michael R Bax diff --git a/Misc/NEWS b/Misc/NEWS index da211014ac3..fedcd35c899 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -130,6 +130,9 @@ Core and Builtins Library ------- +- Issue #16152: fix tokenize to ignore whitespace at the end of the code when + no newline is found. Patch by Ned Batchelder. + - Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu Patch by Todd Rovito.