bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119)

* Allow underscores in numeric literals in lib2to3.
* Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize.
* Add test case for underscores in literals in Python 3.
This commit is contained in:
Nevada Sanchez 2017-04-13 13:32:54 -04:00 committed by Mariatta
parent 873ef20d00
commit a6e395dffa
2 changed files with 30 additions and 8 deletions

View File

@ -54,16 +54,16 @@ Comment = r'#[^\r\n]*'
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Name = r'[a-zA-Z_]\w*'
Binnumber = r'0[bB][01]*'
Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
Octnumber = r'0[oO]?[0-7]*[lL]?'
Decnumber = r'[1-9]\d*[lL]?'
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+'
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Expfloat = r'\d+' + Exponent
Exponent = r'[eE][-+]?\d+(?:_\d+)*'
Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
Expfloat = r'\d+(?:_\d+)*' + Exponent
Floatnumber = group(Pointfloat, Expfloat)
Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
Number = group(Imagnumber, Floatnumber, Intnumber)
# Tail end of ' string.

View File

@ -72,6 +72,28 @@ class TokenTests(unittest.TestCase):
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
def testUnderscoresInNumbers(self):
# Integers
x = 1_0
x = 123_456_7_89
x = 0xabc_123_4_5
x = 0X_abc_123
x = 0B11_01
x = 0b_11_01
x = 0o45_67
x = 0O_45_67
# Floats
x = 3_1.4
x = 03_1.4
x = 3_1.
x = .3_1
x = 3.1_4
x = 0_3.1_4
x = 3e1_4
x = 3_1e+4_1
x = 3_1E-4_1
def testFloats(self):
x = 3.14
x = 314.