cpython/Lib/test/test_source_encoding.py

249 lines
8.7 KiB
Python
Raw Normal View History

Merged revisions 69576,69579-69580,69589,69619-69620,69633,69703-69704,69728-69730 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r69576 | georg.brandl | 2009-02-13 04:56:50 -0600 (Fri, 13 Feb 2009) | 1 line #1661108: note that urlsafe encoded string can contain "=". ........ r69579 | georg.brandl | 2009-02-13 05:06:59 -0600 (Fri, 13 Feb 2009) | 2 lines Fix warnings GCC emits where the argument of PyErr_Format is a single variable. ........ r69580 | georg.brandl | 2009-02-13 05:10:04 -0600 (Fri, 13 Feb 2009) | 2 lines Fix warnings GCC emits where the argument of PyErr_Format is a single variable. ........ r69589 | martin.v.loewis | 2009-02-13 14:11:34 -0600 (Fri, 13 Feb 2009) | 2 lines Move amd64 properties further to the top, so that they override the linker options correctly. ........ r69619 | benjamin.peterson | 2009-02-14 11:00:51 -0600 (Sat, 14 Feb 2009) | 1 line this needn't be a shebang line ........ r69620 | georg.brandl | 2009-02-14 11:01:36 -0600 (Sat, 14 Feb 2009) | 1 line #5179: don't leak PIPE fds when child execution fails. ........ r69633 | hirokazu.yamamoto | 2009-02-15 03:19:48 -0600 (Sun, 15 Feb 2009) | 1 line Fixed typo. ........ r69703 | raymond.hettinger | 2009-02-16 16:42:54 -0600 (Mon, 16 Feb 2009) | 3 lines Issue 5229: Documentation for super() neglects to say what super() actually does ........ r69704 | raymond.hettinger | 2009-02-16 17:00:25 -0600 (Mon, 16 Feb 2009) | 1 line Add explanation for super(type1, type2). ........ r69728 | georg.brandl | 2009-02-17 18:22:55 -0600 (Tue, 17 Feb 2009) | 2 lines #5297: fix example. ........ r69729 | georg.brandl | 2009-02-17 18:25:13 -0600 (Tue, 17 Feb 2009) | 2 lines #5296: sequence -> iterable. ........ r69730 | georg.brandl | 2009-02-17 18:31:36 -0600 (Tue, 17 Feb 2009) | 2 lines #5268: mention VMSError. ........
2009-02-19 00:22:03 -04:00
# -*- coding: koi8-r -*-
import unittest
from test.support import script_helper, captured_stdout, requires_subprocess
from test.support.os_helper import TESTFN, unlink, rmtree
from test.support.import_helper import unload
import importlib
import os
import sys
2014-02-28 10:43:36 -04:00
import subprocess
import tempfile
class MiscSourceEncodingTest(unittest.TestCase):
def test_pep263(self):
self.assertEqual(
"<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>".encode("utf-8"),
b'\xd0\x9f\xd0\xb8\xd1\x82\xd0\xbe\xd0\xbd'
)
self.assertEqual(
"\<EFBFBD>".encode("utf-8"),
b'\\\xd0\x9f'
)
def test_compilestring(self):
# see #1882
c = compile(b"\n# coding: utf-8\nu = '\xc3\xb3'\n", "dummy", "exec")
d = {}
exec(c, d)
self.assertEqual(d['u'], '\xf3')
def test_issue2301(self):
try:
compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
except SyntaxError as v:
self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'")
else:
self.fail()
def test_issue4626(self):
c = compile("# coding=latin-1\n\u00c6 = '\u00c6'", "dummy", "exec")
d = {}
exec(c, d)
self.assertEqual(d['\xc6'], '\xc6')
def test_issue3297(self):
c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec")
d = {}
exec(c, d)
self.assertEqual(d['a'], d['b'])
self.assertEqual(len(d['a']), len(d['b']))
self.assertEqual(ascii(d['a']), ascii(d['b']))
def test_issue7820(self):
# Ensure that check_bom() restores all bytes in the right order if
# check_bom() fails in pydebug mode: a buffer starts with the first
# byte of a valid BOM, but next bytes are different
# one byte in common with the UTF-16-LE BOM
self.assertRaises(SyntaxError, eval, b'\xff\x20')
# one byte in common with the UTF-8 BOM
self.assertRaises(SyntaxError, eval, b'\xef\x20')
# two bytes in common with the UTF-8 BOM
self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20')
@requires_subprocess()
2014-02-28 10:43:36 -04:00
def test_20731(self):
2014-02-28 10:49:06 -04:00
sub = subprocess.Popen([sys.executable,
2014-02-28 10:43:36 -04:00
os.path.join(os.path.dirname(__file__),
'coding20731.py')],
stderr=subprocess.PIPE)
err = sub.communicate()[1]
2014-03-01 11:34:22 -04:00
self.assertEqual(sub.returncode, 0)
self.assertNotIn(b'SyntaxError', err)
2014-02-28 10:43:36 -04:00
def test_error_message(self):
compile(b'# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
with self.assertRaisesRegex(SyntaxError, 'fake'):
compile(b'# -*- coding: fake -*-\n', 'dummy', 'exec')
with self.assertRaisesRegex(SyntaxError, 'iso-8859-15'):
compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n',
'dummy', 'exec')
with self.assertRaisesRegex(SyntaxError, 'BOM'):
compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n',
'dummy', 'exec')
with self.assertRaisesRegex(SyntaxError, 'fake'):
compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
with self.assertRaisesRegex(SyntaxError, 'BOM'):
compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
def test_bad_coding(self):
module_name = 'bad_coding'
self.verify_bad_module(module_name)
def test_bad_coding2(self):
module_name = 'bad_coding2'
self.verify_bad_module(module_name)
def verify_bad_module(self, module_name):
self.assertRaises(SyntaxError, __import__, 'test.' + module_name)
path = os.path.dirname(__file__)
filename = os.path.join(path, module_name + '.py')
with open(filename, "rb") as fp:
bytes = fp.read()
self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')
def test_exec_valid_coding(self):
d = {}
exec(b'# coding: cp949\na = "\xaa\xa7"\n', d)
self.assertEqual(d['a'], '\u3047')
def test_file_parse(self):
# issue1134: all encodings outside latin-1 and utf-8 fail on
# multiline strings and long lines (>512 columns)
unload(TESTFN)
filename = TESTFN + ".py"
f = open(filename, "w", encoding="cp1252")
sys.path.insert(0, os.curdir)
try:
with f:
f.write("# -*- coding: cp1252 -*-\n")
f.write("'''A short string\n")
f.write("'''\n")
f.write("'A very long string %s'\n" % ("X" * 1000))
2012-04-14 16:06:17 -03:00
importlib.invalidate_caches()
__import__(TESTFN)
finally:
del sys.path[0]
unlink(filename)
unlink(filename + "c")
unlink(filename + "o")
unload(TESTFN)
rmtree('__pycache__')
Merged revisions 73004,73439,73496,73509,73529,73564,73576-73577,73595-73596,73605 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r73004 | jeffrey.yasskin | 2009-05-28 22:44:31 -0500 (Thu, 28 May 2009) | 5 lines Fix nearly all compilation warnings under Apple gcc-4.0. Tested with OPT="-g -Wall -Wstrict-prototypes -Werror" in both --with-pydebug mode and --without. There's still a batch of non-prototype warnings in Xlib.h that I don't know how to fix. ........ r73439 | benjamin.peterson | 2009-06-15 19:29:31 -0500 (Mon, 15 Jun 2009) | 1 line don't mask encoding errors when decoding a string #6289 ........ r73496 | vinay.sajip | 2009-06-21 12:37:27 -0500 (Sun, 21 Jun 2009) | 1 line Issue #6314: logging.basicConfig() performs extra checks on the "level" argument. ........ r73509 | amaury.forgeotdarc | 2009-06-22 14:33:48 -0500 (Mon, 22 Jun 2009) | 2 lines #4490 Fix sample code run by "python -m xml.sax.xmlreader" ........ r73529 | r.david.murray | 2009-06-23 13:02:46 -0500 (Tue, 23 Jun 2009) | 4 lines Fix issue 5230 by having pydoc's safeimport check to see if the import error was thrown from itself in order to decide if the module can't be found. Thanks to Lucas Prado Melo for collaborating on the fix and tests. ........ r73564 | amaury.forgeotdarc | 2009-06-25 17:29:29 -0500 (Thu, 25 Jun 2009) | 6 lines #2016 Fix a crash in function call when the **kwargs dictionary is mutated during the function call setup. This even gives a slight speedup, probably because tuple allocation is faster than PyMem_NEW. ........ r73576 | benjamin.peterson | 2009-06-26 18:37:06 -0500 (Fri, 26 Jun 2009) | 1 line document is_declared_global() ........ r73577 | benjamin.peterson | 2009-06-27 09:16:23 -0500 (Sat, 27 Jun 2009) | 1 line link to extensive generator docs in the reference manual ........ r73595 | ezio.melotti | 2009-06-27 18:45:39 -0500 (Sat, 27 Jun 2009) | 1 line stmt and setup can contain multiple statements, see #5896 ........ r73596 | ezio.melotti | 2009-06-27 19:07:45 -0500 (Sat, 27 Jun 2009) | 1 line Fixed a wrong apostrophe ........ r73605 | georg.brandl | 2009-06-28 07:10:18 -0500 (Sun, 28 Jun 2009) | 1 line Remove stray pychecker directive. ........
2009-06-28 14:22:03 -03:00
def test_error_from_string(self):
# See http://bugs.python.org/issue6289
input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8')
with self.assertRaises(SyntaxError) as c:
Merged revisions 73004,73439,73496,73509,73529,73564,73576-73577,73595-73596,73605 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r73004 | jeffrey.yasskin | 2009-05-28 22:44:31 -0500 (Thu, 28 May 2009) | 5 lines Fix nearly all compilation warnings under Apple gcc-4.0. Tested with OPT="-g -Wall -Wstrict-prototypes -Werror" in both --with-pydebug mode and --without. There's still a batch of non-prototype warnings in Xlib.h that I don't know how to fix. ........ r73439 | benjamin.peterson | 2009-06-15 19:29:31 -0500 (Mon, 15 Jun 2009) | 1 line don't mask encoding errors when decoding a string #6289 ........ r73496 | vinay.sajip | 2009-06-21 12:37:27 -0500 (Sun, 21 Jun 2009) | 1 line Issue #6314: logging.basicConfig() performs extra checks on the "level" argument. ........ r73509 | amaury.forgeotdarc | 2009-06-22 14:33:48 -0500 (Mon, 22 Jun 2009) | 2 lines #4490 Fix sample code run by "python -m xml.sax.xmlreader" ........ r73529 | r.david.murray | 2009-06-23 13:02:46 -0500 (Tue, 23 Jun 2009) | 4 lines Fix issue 5230 by having pydoc's safeimport check to see if the import error was thrown from itself in order to decide if the module can't be found. Thanks to Lucas Prado Melo for collaborating on the fix and tests. ........ r73564 | amaury.forgeotdarc | 2009-06-25 17:29:29 -0500 (Thu, 25 Jun 2009) | 6 lines #2016 Fix a crash in function call when the **kwargs dictionary is mutated during the function call setup. This even gives a slight speedup, probably because tuple allocation is faster than PyMem_NEW. ........ r73576 | benjamin.peterson | 2009-06-26 18:37:06 -0500 (Fri, 26 Jun 2009) | 1 line document is_declared_global() ........ r73577 | benjamin.peterson | 2009-06-27 09:16:23 -0500 (Sat, 27 Jun 2009) | 1 line link to extensive generator docs in the reference manual ........ r73595 | ezio.melotti | 2009-06-27 18:45:39 -0500 (Sat, 27 Jun 2009) | 1 line stmt and setup can contain multiple statements, see #5896 ........ r73596 | ezio.melotti | 2009-06-27 19:07:45 -0500 (Sat, 27 Jun 2009) | 1 line Fixed a wrong apostrophe ........ r73605 | georg.brandl | 2009-06-28 07:10:18 -0500 (Sun, 28 Jun 2009) | 1 line Remove stray pychecker directive. ........
2009-06-28 14:22:03 -03:00
compile(input, "<string>", "exec")
expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \
"ordinal not in range(128)"
self.assertTrue(c.exception.args[0].startswith(expected),
msg=c.exception.args[0])
Merged revisions 73004,73439,73496,73509,73529,73564,73576-73577,73595-73596,73605 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r73004 | jeffrey.yasskin | 2009-05-28 22:44:31 -0500 (Thu, 28 May 2009) | 5 lines Fix nearly all compilation warnings under Apple gcc-4.0. Tested with OPT="-g -Wall -Wstrict-prototypes -Werror" in both --with-pydebug mode and --without. There's still a batch of non-prototype warnings in Xlib.h that I don't know how to fix. ........ r73439 | benjamin.peterson | 2009-06-15 19:29:31 -0500 (Mon, 15 Jun 2009) | 1 line don't mask encoding errors when decoding a string #6289 ........ r73496 | vinay.sajip | 2009-06-21 12:37:27 -0500 (Sun, 21 Jun 2009) | 1 line Issue #6314: logging.basicConfig() performs extra checks on the "level" argument. ........ r73509 | amaury.forgeotdarc | 2009-06-22 14:33:48 -0500 (Mon, 22 Jun 2009) | 2 lines #4490 Fix sample code run by "python -m xml.sax.xmlreader" ........ r73529 | r.david.murray | 2009-06-23 13:02:46 -0500 (Tue, 23 Jun 2009) | 4 lines Fix issue 5230 by having pydoc's safeimport check to see if the import error was thrown from itself in order to decide if the module can't be found. Thanks to Lucas Prado Melo for collaborating on the fix and tests. ........ r73564 | amaury.forgeotdarc | 2009-06-25 17:29:29 -0500 (Thu, 25 Jun 2009) | 6 lines #2016 Fix a crash in function call when the **kwargs dictionary is mutated during the function call setup. This even gives a slight speedup, probably because tuple allocation is faster than PyMem_NEW. ........ r73576 | benjamin.peterson | 2009-06-26 18:37:06 -0500 (Fri, 26 Jun 2009) | 1 line document is_declared_global() ........ r73577 | benjamin.peterson | 2009-06-27 09:16:23 -0500 (Sat, 27 Jun 2009) | 1 line link to extensive generator docs in the reference manual ........ r73595 | ezio.melotti | 2009-06-27 18:45:39 -0500 (Sat, 27 Jun 2009) | 1 line stmt and setup can contain multiple statements, see #5896 ........ r73596 | ezio.melotti | 2009-06-27 19:07:45 -0500 (Sat, 27 Jun 2009) | 1 line Fixed a wrong apostrophe ........ r73605 | georg.brandl | 2009-06-28 07:10:18 -0500 (Sun, 28 Jun 2009) | 1 line Remove stray pychecker directive. ........
2009-06-28 14:22:03 -03:00
class AbstractSourceEncodingTest:
def test_default_coding(self):
src = (b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
def test_first_coding_line(self):
src = (b'#coding:iso8859-15\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_second_coding_line(self):
src = (b'#\n'
b'#coding:iso8859-15\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_third_coding_line(self):
# Only first two lines are tested for a magic comment.
src = (b'#\n'
b'#\n'
b'#coding:iso8859-15\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
def test_double_coding_line(self):
# If the first line matches the second line is ignored.
src = (b'#coding:iso8859-15\n'
b'#coding:latin1\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_double_coding_same_line(self):
src = (b'#coding:iso8859-15 coding:latin1\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_first_non_utf8_coding_line(self):
src = (b'#coding:iso-8859-15 \xa4\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_second_non_utf8_coding_line(self):
src = (b'\n'
b'#coding:iso-8859-15 \xa4\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
def test_utf8_bom(self):
src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
def test_utf8_bom_and_utf8_coding_line(self):
src = (b'\xef\xbb\xbf#coding:utf-8\n'
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
def test_crlf(self):
src = (b'print(ascii("""\r\n"""))\n')
out = self.check_script_output(src, br"'\n'")
def test_crcrlf(self):
src = (b'print(ascii("""\r\r\n"""))\n')
out = self.check_script_output(src, br"'\n\n'")
def test_crcrcrlf(self):
src = (b'print(ascii("""\r\r\r\n"""))\n')
out = self.check_script_output(src, br"'\n\n\n'")
def test_crcrcrlf2(self):
src = (b'#coding:iso-8859-1\n'
b'print(ascii("""\r\r\r\n"""))\n')
out = self.check_script_output(src, br"'\n\n\n'")
class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
def check_script_output(self, src, expected):
with captured_stdout() as stdout:
exec(src)
out = stdout.getvalue().encode('latin1')
self.assertEqual(out.rstrip(), expected)
class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
def check_script_output(self, src, expected):
with tempfile.TemporaryDirectory() as tmpd:
fn = os.path.join(tmpd, 'test.py')
with open(fn, 'wb') as fp:
fp.write(src)
res = script_helper.assert_python_ok(fn)
self.assertEqual(res.out.rstrip(), expected)
if __name__ == "__main__":
unittest.main()