cpython/Lib/test/test_tokenize.py

101 lines
2.3 KiB
Python
Raw Normal View History

from test.test_support import verbose, findfile, is_resource_enabled, TestFailed
import os, glob, random
from tokenize import (tokenize, generate_tokens, untokenize,
NUMBER, NAME, OP, STRING)
1997-10-27 18:15:06 -04:00
if verbose:
print 'starting...'
f = file(findfile('tokenize_tests' + os.extsep + 'txt'))
tokenize(f.readline)
f.close()
###### Test roundtrip for untokenize ##########################
def test_roundtrip(f):
## print 'Testing:', f
f = file(f)
try:
fulltok = list(generate_tokens(f.readline))
finally:
f.close()
t1 = [tok[:2] for tok in fulltok]
newtext = untokenize(t1)
readline = iter(newtext.splitlines(1)).next
t2 = [tok[:2] for tok in generate_tokens(readline)]
assert t1 == t2
f = findfile('tokenize_tests' + os.extsep + 'txt')
test_roundtrip(f)
testdir = os.path.dirname(f) or os.curdir
testfiles = glob.glob(testdir + os.sep + 'test*.py')
if not is_resource_enabled('compiler'):
testfiles = random.sample(testfiles, 10)
for f in testfiles:
test_roundtrip(f)
###### Test detecton of IndentationError ######################
from cStringIO import StringIO
sampleBadText = """
def foo():
bar
baz
"""
try:
for tok in generate_tokens(StringIO(sampleBadText).readline):
pass
except IndentationError:
pass
else:
raise TestFailed("Did not detect IndentationError:")
###### Test example in the docs ###############################
from decimal import Decimal
from cStringIO import StringIO
def decistmt(s):
"""Substitute Decimals for floats in a string of statements.
>>> from decimal import Decimal
>>> s = 'print +21.3e-5*-.1234/81.7'
>>> decistmt(s)
"print +Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')"
>>> exec(s)
-3.21716034272e-007
>>> exec(decistmt(s))
-3.217160342717258261933904529E-7
"""
result = []
g = generate_tokens(StringIO(s).readline) # tokenize the string
for toknum, tokval, _, _, _ in g:
if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens
result.extend([
(NAME, 'Decimal'),
(OP, '('),
(STRING, repr(tokval)),
(OP, ')')
])
else:
result.append((toknum, tokval))
return untokenize(result)
import doctest
doctest.testmod()
1997-10-27 18:15:06 -04:00
if verbose:
print 'finished'