Merge with 3.3

This commit is contained in:
Terry Jan Reedy 2014-02-17 23:12:37 -05:00
commit 79bf89986c
3 changed files with 27 additions and 13 deletions

View File

@ -1165,6 +1165,19 @@ class UntokenizeTest(TestCase):
'start (1,3) precedes previous end (2,2)')
self.assertRaises(ValueError, u.add_whitespace, (2,1))
def test_iter_compat(self):
u = Untokenizer()
token = (NAME, 'Hello')
tokens = [(ENCODING, 'utf-8'), token]
u.compat(token, iter([]))
self.assertEqual(u.tokens, ["Hello "])
u = Untokenizer()
self.assertEqual(u.untokenize(iter([token])), 'Hello ')
u = Untokenizer()
self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
self.assertEqual(u.encoding, 'utf-8')
self.assertEqual(untokenize(iter(tokens)), b'Hello ')
__test__ = {"doctests" : doctests, 'decistmt': decistmt}

View File

@ -25,12 +25,14 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
'Michael Foord')
import builtins
import re
import sys
from token import *
from codecs import lookup, BOM_UTF8
import collections
from io import TextIOWrapper
from itertools import chain
import re
import sys
from token import *
cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
@ -237,9 +239,10 @@ class Untokenizer:
self.tokens.append(" " * col_offset)
def untokenize(self, iterable):
for t in iterable:
it = iter(iterable)
for t in it:
if len(t) == 2:
self.compat(t, iterable)
self.compat(t, it)
break
tok_type, token, start, end, line = t
if tok_type == ENCODING:
@ -254,17 +257,12 @@ class Untokenizer:
return "".join(self.tokens)
def compat(self, token, iterable):
startline = False
indents = []
toks_append = self.tokens.append
toknum, tokval = token
if toknum in (NAME, NUMBER):
tokval += ' '
if toknum in (NEWLINE, NL):
startline = True
startline = token[0] in (NEWLINE, NL)
prevstring = False
for tok in iterable:
for tok in chain([token], iterable):
toknum, tokval = tok[:2]
if toknum == ENCODING:
self.encoding = tokval

View File

@ -40,6 +40,9 @@ Library
- Issue #17671: Fixed a crash when use non-initialized io.BufferedRWPair.
Based on patch by Stephen Tu.
- Issue #8478: Untokenizer.compat processes first token from iterator input.
Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.
- Issue #20594: Avoid name clash with the libc function posix_close.
- Issue #19856: shutil.move() failed to move a directory to other directory