From 87c0f1d1c95eb1766a1340cd100533dfd7a86d52 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 19 Nov 2007 18:03:44 +0000 Subject: [PATCH] Merged revisions 59041-59055 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r59044 | neal.norwitz | 2007-11-18 17:46:20 -0800 (Sun, 18 Nov 2007) | 1 line Use a slightly more recent version than 1.5.2b2. ........ r59047 | walter.doerwald | 2007-11-19 04:14:05 -0800 (Mon, 19 Nov 2007) | 2 lines Fix typo in comment. ........ r59049 | walter.doerwald | 2007-11-19 04:41:10 -0800 (Mon, 19 Nov 2007) | 4 lines Fix for #1444: utf_8_sig.StreamReader was (indirectly through decode()) calling codecs.utf_8_decode() with final==True, which falled with incomplete byte sequences. Fix and test by James G. Sack. ........ r59051 | nick.coghlan | 2007-11-19 05:56:27 -0800 (Mon, 19 Nov 2007) | 1 line Enable some test_cmd_line_script debugging output to investigate failure on Mac OSX buildbot ........ r59053 | facundo.batista | 2007-11-19 08:30:24 -0800 (Mon, 19 Nov 2007) | 3 lines Fixed detail in add_type() explanation (issue 1463). ........ r59054 | guido.van.rossum | 2007-11-19 09:35:24 -0800 (Mon, 19 Nov 2007) | 2 lines Make this work stand-alone, too. ........ r59055 | guido.van.rossum | 2007-11-19 09:50:22 -0800 (Mon, 19 Nov 2007) | 3 lines Fix the OSX failures in this test -- they were due to /tmp being a symlink to /private/tmp. Adding a call to os.path.realpath() to temp_dir() fixed it. ........ --- Doc/library/mimetypes.rst | 4 +-- Lib/encodings/utf_8_sig.py | 16 +++++++--- Lib/test/test_cmd_line_script.py | 9 +++--- Lib/test/test_codecs.py | 53 +++++++++++++++++++++++++++++--- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 50140c6dae8..f59cd829a91 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -96,8 +96,8 @@ behavior of the module. extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. - When *strict* is the mapping will added to the official MIME types, otherwise to - the non-standard ones. + When *strict* is True (the default), the mapping will added to the official MIME + types, otherwise to the non-standard ones. .. data:: inited diff --git a/Lib/encodings/utf_8_sig.py b/Lib/encodings/utf_8_sig.py index 07cd5eeb1b2..1bb479203f3 100644 --- a/Lib/encodings/utf_8_sig.py +++ b/Lib/encodings/utf_8_sig.py @@ -103,12 +103,18 @@ class StreamReader(codecs.StreamReader): pass def decode(self, input, errors='strict'): - if len(input) < 3 and codecs.BOM_UTF8.startswith(input): - # not enough data to decide if this is a BOM - # => try again on the next call - return ("", 0) + if len(input) < 3: + if codecs.BOM_UTF8.startswith(input): + # not enough data to decide if this is a BOM + # => try again on the next call + return ("", 0) + elif input[:3] == codecs.BOM_UTF8: + self.decode = codecs.utf_8_decode + (output, consumed) = codecs.utf_8_decode(input[3:],errors) + return (output, consumed+3) + # (else) no BOM present self.decode = codecs.utf_8_decode - return decode(input, errors) + return codecs.utf_8_decode(input, errors) ### encodings module API diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 3c6e4a0af59..dcb768f3c0a 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -29,6 +29,7 @@ def _run_python(*args): @contextlib.contextmanager def temp_dir(): dirname = tempfile.mkdtemp() + dirname = os.path.realpath(dirname) try: yield dirname finally: @@ -82,7 +83,7 @@ def _make_test_zip(zip_dir, zip_basename, script_name): zip_file.close() # if verbose: # zip_file = zipfile.ZipFile(zip_name, 'r') - # print "Contents of %r:" % zip_name + # print("Contents of %r:" % zip_name) # zip_file.printdir() # zip_file.close() return zip_name @@ -90,9 +91,9 @@ def _make_test_zip(zip_dir, zip_basename, script_name): class CmdLineTest(unittest.TestCase): def _check_script(self, script_name, expected_file, expected_argv0): exit_code, data = _run_python(script_name) - # if verbose: - # print "Output from test script %r:" % script_name - # print data + if verbose: + print("Output from test script %r:" % script_name) + print(data) self.assertEqual(exit_code, 0, data) printed_file = '__file__==%r' % expected_file printed_argv0 = 'sys.argv[0]==%r' % expected_argv0 diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 5833c6db183..413a5aa8d27 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -59,7 +59,7 @@ class MixInCheckStateHandling: class ReadTest(unittest.TestCase, MixInCheckStateHandling): def check_partial(self, input, partialresults): # get a StreamReader for the encoding and feed the bytestring version - # of input to the reader byte by byte. Read every available from + # of input to the reader byte by byte. Read everything available from # the StreamReader and check that the results equal the appropriate # entries from partialresults. q = Queue(b"") @@ -618,10 +618,53 @@ class UTF8SigTest(ReadTest): s = "spam" self.assertEqual(d.decode(s.encode("utf-8-sig")), s) - def test_decoder_state(self): - u = "\x00\x7f\x80\xff\u0100\u07ff\u0800\uffff\U0010ffff" - self.check_state_handling_decode(self.encoding, - u, u.encode(self.encoding)) + def test_stream_bom(self): + unistring = "ABC\u00A1\u2200XYZ" + bytestring = codecs.BOM_UTF8 + b"ABC\xC2\xA1\xE2\x88\x80XYZ" + + reader = codecs.getreader("utf-8-sig") + for sizehint in [None] + list(range(1, 11)) + \ + [64, 128, 256, 512, 1024]: + istream = reader(io.BytesIO(bytestring)) + ostream = io.StringIO() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + ostream.write(data) + + got = ostream.getvalue() + self.assertEqual(got, unistring) + + def test_stream_bare(self): + unistring = "ABC\u00A1\u2200XYZ" + bytestring = b"ABC\xC2\xA1\xE2\x88\x80XYZ" + + reader = codecs.getreader("utf-8-sig") + for sizehint in [None] + list(range(1, 11)) + \ + [64, 128, 256, 512, 1024]: + istream = reader(io.BytesIO(bytestring)) + ostream = io.StringIO() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + ostream.write(data) + + got = ostream.getvalue() + self.assertEqual(got, unistring) + +class EscapeDecodeTest(unittest.TestCase): + def test_empty(self): + self.assertEquals(codecs.escape_decode(""), ("", 0)) class RecodingTest(unittest.TestCase): def test_recoding(self):