From 447dc1565826c879faf544cda4bdd62546545166 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 15 Oct 2009 01:49:37 +0000 Subject: [PATCH] use floor division and add a test that exercises the tabsize codepath --- Lib/test/test_tokenize.py | 17 +++++++++++++++++ Lib/tokenize.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ae5f410b7a1..e5371cfc42a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -508,6 +508,23 @@ pass the '-ucompiler' option to process the full directory. ... break ... else: True True + +Evil tabs + >>> dump_tokens("def f():\\n\\tif x\\n \tpass") + NAME 'def' (1, 0) (1, 3) + NAME 'f' (1, 4) (1, 5) + OP '(' (1, 5) (1, 6) + OP ')' (1, 6) (1, 7) + OP ':' (1, 7) (1, 8) + NEWLINE '\\n' (1, 8) (1, 9) + INDENT '\\t' (2, 0) (2, 1) + NAME 'if' (2, 1) (2, 3) + NAME 'x' (2, 4) (2, 5) + NEWLINE '\\n' (2, 5) (2, 6) + INDENT ' ' (3, 0) (3, 9) + NAME 'pass' (3, 9) (3, 13) + DEDENT '' (4, 0) (4, 0) + DEDENT '' (4, 0) (4, 0) """ diff --git a/Lib/tokenize.py b/Lib/tokenize.py index ad3cf9d1626..686ad66f6dd 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -319,7 +319,7 @@ def generate_tokens(readline): if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': - column = (column/tabsize + 1)*tabsize + column = (column//tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: