#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.

This commit is contained in:
Ezio Melotti 2012-11-03 17:38:43 +02:00
parent 4552e3f95c
commit 2cc3b4ba9f
4 changed files with 12 additions and 1 deletions

View File

@ -552,6 +552,11 @@ Evil tabs
DEDENT '' (4, 0) (4, 0)
DEDENT '' (4, 0) (4, 0)
Pathological whitespace (http://bugs.python.org/issue16152)
>>> dump_tokens("@ ")
ENCODING 'utf-8' (0, 0) (0, 0)
OP '@' (1, 0) (1, 1)
Non-ascii identifiers
>>> dump_tokens("Örter = 'places'\\ngrün = 'green'")

View File

@ -108,7 +108,7 @@ ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
group("'", r'\\\r?\n'),
r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
def _compile(expr):
@ -473,6 +473,8 @@ def _tokenize(readline, encoding):
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
if start == end:
continue
token, initial = line[start:end], line[start]
if (initial in numchars or # ordinary number

View File

@ -71,6 +71,7 @@ Des Barry
Ulf Bartelt
Don Bashford
Nick Bastin
Ned Batchelder
Jeff Bauer
Mike Bayer
Michael R Bax

View File

@ -143,6 +143,9 @@ Core and Builtins
Library
-------
- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
no newline is found. Patch by Ned Batchelder.
- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
Patch by Todd Rovito.