#16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.
This commit is contained in:
parent
4552e3f95c
commit
2cc3b4ba9f
|
@ -552,6 +552,11 @@ Evil tabs
|
|||
DEDENT '' (4, 0) (4, 0)
|
||||
DEDENT '' (4, 0) (4, 0)
|
||||
|
||||
Pathological whitespace (http://bugs.python.org/issue16152)
|
||||
>>> dump_tokens("@ ")
|
||||
ENCODING 'utf-8' (0, 0) (0, 0)
|
||||
OP '@' (1, 0) (1, 1)
|
||||
|
||||
Non-ascii identifiers
|
||||
|
||||
>>> dump_tokens("Örter = 'places'\\ngrün = 'green'")
|
||||
|
|
|
@ -108,7 +108,7 @@ ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
|
|||
group("'", r'\\\r?\n'),
|
||||
r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
|
||||
group('"', r'\\\r?\n'))
|
||||
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
|
||||
PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
|
||||
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
|
||||
|
||||
def _compile(expr):
|
||||
|
@ -473,6 +473,8 @@ def _tokenize(readline, encoding):
|
|||
if pseudomatch: # scan for tokens
|
||||
start, end = pseudomatch.span(1)
|
||||
spos, epos, pos = (lnum, start), (lnum, end), end
|
||||
if start == end:
|
||||
continue
|
||||
token, initial = line[start:end], line[start]
|
||||
|
||||
if (initial in numchars or # ordinary number
|
||||
|
|
|
@ -71,6 +71,7 @@ Des Barry
|
|||
Ulf Bartelt
|
||||
Don Bashford
|
||||
Nick Bastin
|
||||
Ned Batchelder
|
||||
Jeff Bauer
|
||||
Mike Bayer
|
||||
Michael R Bax
|
||||
|
|
Loading…
Reference in New Issue