From c28f6fa505f717698038b41492265129c77dfc0e Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 16 Mar 2013 19:48:51 +0200 Subject: [PATCH 1/5] #11420: make test suite pass with -B/DONTWRITEBYTECODE set. Initial patch by Thomas Wouters. --- Lib/distutils/tests/test_bdist_dumb.py | 6 +-- Lib/importlib/test/source/test_file_loader.py | 3 +- Lib/test/test_imp.py | 8 ++-- Lib/test/test_import.py | 13 +++++ Lib/test/test_runpy.py | 48 ++++++++++--------- Misc/NEWS | 3 ++ 6 files changed, 52 insertions(+), 29 deletions(-) diff --git a/Lib/distutils/tests/test_bdist_dumb.py b/Lib/distutils/tests/test_bdist_dumb.py index 1037d8216ec..0ad32d421ec 100644 --- a/Lib/distutils/tests/test_bdist_dumb.py +++ b/Lib/distutils/tests/test_bdist_dumb.py @@ -88,9 +88,9 @@ class BuildDumbTestCase(support.TempdirManager, fp.close() contents = sorted(os.path.basename(fn) for fn in contents) - wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], - 'foo.%s.pyc' % imp.get_tag(), - 'foo.py'] + wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py'] + if not sys.dont_write_bytecode: + wanted.append('foo.%s.pyc' % imp.get_tag()) self.assertEqual(contents, sorted(wanted)) def test_suite(): diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index c7a7d8fbcae..b4d7e7dba9e 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -127,7 +127,8 @@ class SimpleTest(unittest.TestCase): finally: os.unlink(file_path) pycache = os.path.dirname(imp.cache_from_source(file_path)) - shutil.rmtree(pycache) + if os.path.exists(pycache): + shutil.rmtree(pycache) def test_timestamp_overflow(self): # When a modification timestamp is larger than 2**32, it should be diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 551ad1b85cc..5df89153990 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -153,9 +153,11 @@ class ImportTests(unittest.TestCase): mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') self.assertEqual(mod.a, 1) - mod = imp.load_compiled( - temp_mod_name, imp.cache_from_source(temp_mod_name + '.py')) - self.assertEqual(mod.a, 1) + if not sys.dont_write_bytecode: + mod = imp.load_compiled( + temp_mod_name, + imp.cache_from_source(temp_mod_name + '.py')) + self.assertEqual(mod.a, 1) if not os.path.exists(test_package_name): os.mkdir(test_package_name) diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index b10f350e1e9..1aa665a1b26 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -20,6 +20,10 @@ from test.support import ( from test import script_helper +skip_if_dont_write_bytecode = unittest.skipIf( + sys.dont_write_bytecode, + "test meaningful only when writing bytecode") + def _files(name): return (name + os.extsep + "py", name + os.extsep + "pyc", @@ -109,6 +113,7 @@ class ImportTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") + @skip_if_dont_write_bytecode def test_execute_bit_not_copied(self): # Issue 6070: under posix .pyc files got their execute bit set if # the .py file had the execute bit set, but they aren't executable. @@ -133,6 +138,7 @@ class ImportTests(unittest.TestCase): remove_files(TESTFN) unload(TESTFN) + @skip_if_dont_write_bytecode def test_rewrite_pyc_with_read_only_source(self): # Issue 6074: a long time ago on posix, and more recently on Windows, # a read only source file resulted in a read only pyc file, which @@ -299,6 +305,7 @@ class ImportTests(unittest.TestCase): remove_files(TESTFN) unload(TESTFN) + @skip_if_dont_write_bytecode def test_file_to_source(self): # check if __file__ points to the source file where available source = TESTFN + ".py" @@ -619,6 +626,7 @@ class PycacheTests(unittest.TestCase): del sys.path[0] self._clean() + @skip_if_dont_write_bytecode def test_import_pyc_path(self): self.assertFalse(os.path.exists('__pycache__')) __import__(TESTFN) @@ -631,6 +639,7 @@ class PycacheTests(unittest.TestCase): "test meaningful only on posix systems") @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "due to varying filesystem permission semantics (issue #11956)") + @skip_if_dont_write_bytecode def test_unwritable_directory(self): # When the umask causes the new __pycache__ directory to be # unwritable, the import still succeeds but no .pyc file is written. @@ -640,6 +649,7 @@ class PycacheTests(unittest.TestCase): self.assertFalse(os.path.exists(os.path.join( '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag)))) + @skip_if_dont_write_bytecode def test_missing_source(self): # With PEP 3147 cache layout, removing the source but leaving the pyc # file does not satisfy the import. @@ -650,6 +660,7 @@ class PycacheTests(unittest.TestCase): forget(TESTFN) self.assertRaises(ImportError, __import__, TESTFN) + @skip_if_dont_write_bytecode def test_missing_source_legacy(self): # Like test_missing_source() except that for backward compatibility, # when the pyc file lives where the py file would have been (and named @@ -670,6 +681,7 @@ class PycacheTests(unittest.TestCase): pyc_file = imp.cache_from_source(TESTFN + '.py') self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file)) + @skip_if_dont_write_bytecode def test___cached___legacy_pyc(self): # Like test___cached__() except that for backward compatibility, # when the pyc file lives where the py file would have been (and named @@ -684,6 +696,7 @@ class PycacheTests(unittest.TestCase): self.assertEqual(m.__cached__, os.path.join(os.curdir, os.path.relpath(pyc_file))) + @skip_if_dont_write_bytecode def test_package___cached__(self): # Like test___cached__ but for packages. def cleanup(): diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 96d2bebbc44..6f0a840f3bb 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -254,11 +254,12 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin): self.check_code_execution(create_ns, expected_ns) __import__(mod_name) os.remove(mod_fname) - make_legacy_pyc(mod_fname) - unload(mod_name) # In case loader caches paths - if verbose > 1: print("Running from compiled:", mod_name) - self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) - self.check_code_execution(create_ns, expected_ns) + if not sys.dont_write_bytecode: + make_legacy_pyc(mod_fname) + unload(mod_name) # In case loader caches paths + if verbose > 1: print("Running from compiled:", mod_name) + self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) + self.check_code_execution(create_ns, expected_ns) finally: self._del_pkg(pkg_dir, depth, mod_name) if verbose > 1: print("Module executed successfully") @@ -287,11 +288,12 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin): self.check_code_execution(create_ns, expected_ns) __import__(mod_name) os.remove(mod_fname) - make_legacy_pyc(mod_fname) - unload(mod_name) # In case loader caches paths - if verbose > 1: print("Running from compiled:", pkg_name) - self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) - self.check_code_execution(create_ns, expected_ns) + if not sys.dont_write_bytecode: + make_legacy_pyc(mod_fname) + unload(mod_name) # In case loader caches paths + if verbose > 1: print("Running from compiled:", pkg_name) + self._fix_ns_for_legacy_pyc(expected_ns, alter_sys) + self.check_code_execution(create_ns, expected_ns) finally: self._del_pkg(pkg_dir, depth, pkg_name) if verbose > 1: print("Package executed successfully") @@ -345,15 +347,16 @@ from ..uncle.cousin import nephew del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) - make_legacy_pyc(mod_fname) - unload(mod_name) # In case the loader caches paths - if verbose > 1: print("Running from compiled:", mod_name) - d2 = run_module(mod_name, run_name=run_name) # Read from bytecode - self.assertEqual(d2["__name__"], expected_name) - self.assertEqual(d2["__package__"], pkg_name) - self.assertIn("sibling", d2) - self.assertIn("nephew", d2) - del d2 # Ensure __loader__ entry doesn't keep file open + if not sys.dont_write_bytecode: + make_legacy_pyc(mod_fname) + unload(mod_name) # In case the loader caches paths + if verbose > 1: print("Running from compiled:", mod_name) + d2 = run_module(mod_name, run_name=run_name) # Read from bytecode + self.assertEqual(d2["__name__"], expected_name) + self.assertEqual(d2["__package__"], pkg_name) + self.assertIn("sibling", d2) + self.assertIn("nephew", d2) + del d2 # Ensure __loader__ entry doesn't keep file open finally: self._del_pkg(pkg_dir, depth, mod_name) if verbose > 1: print("Module executed successfully") @@ -472,9 +475,10 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin): script_name = self._make_test_script(script_dir, mod_name) compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - legacy_pyc = make_legacy_pyc(script_name) - self._check_script(script_dir, "", legacy_pyc, - script_dir) + if not sys.dont_write_bytecode: + legacy_pyc = make_legacy_pyc(script_name) + self._check_script(script_dir, "", legacy_pyc, + script_dir) def test_directory_error(self): with temp_dir() as script_dir: diff --git a/Misc/NEWS b/Misc/NEWS index d2d399eead3..6bde852f075 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -964,6 +964,9 @@ Extension Modules Tests ----- +- Issue #11420: make test suite pass with -B/DONTWRITEBYTECODE set. + Initial patch by Thomas Wouters. + - Issue #10652: make tcl/tk tests run after __all__ test, patch by Zachary Ware. From dff04f44bc0a6f2964b71581934fb1d3f7335fd1 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sat, 16 Mar 2013 15:56:27 -0400 Subject: [PATCH 2/5] Issue #17418: specify that buffer sizes are bytes as soon as possible. --- Doc/library/functions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 99f8335fd92..93d35f84afb 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -868,8 +868,8 @@ are always available. They are listed here in alphabetical order. *buffering* is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size - of a fixed-size chunk buffer. When no *buffering* argument is given, the - default buffering policy works as follows: + in bytes of a fixed-size chunk buffer. When no *buffering* argument is + given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer is chosen using a heuristic trying to determine the underlying device's "block From 2acd2930779b817237b5ecc623f8fc9869ce05d6 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 16 Mar 2013 22:23:30 +0200 Subject: [PATCH 3/5] Fix markup/wording. --- Doc/library/http.server.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 3154f8ea72e..53139a22398 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -181,7 +181,7 @@ of which this module provides three different variants: .. versionchanged:: 3.4 The error response includes a Content-Length header. - explain argument was added. + Added the *explain* argument. .. method:: send_response(code, message=None) From 0f606a636aabe2559525697d0df54ee347f947c0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Mar 2013 22:52:09 +0200 Subject: [PATCH 4/5] Issue #16564: Fixed a performance regression relative to Python 3.1 in the caching of compiled regular expressions. --- Lib/re.py | 34 +++++++++++++++++++++++++--------- Misc/NEWS | 3 +++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Lib/re.py b/Lib/re.py index 85c5a570249..b9106061c85 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -215,8 +215,8 @@ def compile(pattern, flags=0): def purge(): "Clear the regular expression caches" - _compile_typed.cache_clear() - _compile_repl.cache_clear() + _cache.clear() + _cache_repl.clear() def template(pattern, flags=0): "Compile a template pattern, returning a pattern object" @@ -257,14 +257,19 @@ def escape(pattern): # -------------------------------------------------------------------- # internals +_cache = {} +_cache_repl = {} + _pattern_type = type(sre_compile.compile("", 0)) -def _compile(pattern, flags): - return _compile_typed(type(pattern), pattern, flags) +_MAXCACHE = 512 -@functools.lru_cache(maxsize=500) -def _compile_typed(text_bytes_type, pattern, flags): +def _compile(pattern, flags): # internal: compile pattern + try: + return _cache[type(pattern), pattern, flags] + except KeyError: + pass if isinstance(pattern, _pattern_type): if flags: raise ValueError( @@ -272,12 +277,23 @@ def _compile_typed(text_bytes_type, pattern, flags): return pattern if not sre_compile.isstring(pattern): raise TypeError("first argument must be string or compiled pattern") - return sre_compile.compile(pattern, flags) + p = sre_compile.compile(pattern, flags) + if len(_cache) >= _MAXCACHE: + _cache.clear() + _cache[type(pattern), pattern, flags] = p + return p -@functools.lru_cache(maxsize=500) def _compile_repl(repl, pattern): # internal: compile replacement pattern - return sre_parse.parse_template(repl, pattern) + try: + return _cache_repl[repl, pattern] + except KeyError: + pass + p = sre_parse.parse_template(repl, pattern) + if len(_cache_repl) >= _MAXCACHE: + _cache_repl.clear() + _cache_repl[repl, pattern] = p + return p def _expand(pattern, match, template): # internal: match.expand implementation hook diff --git a/Misc/NEWS b/Misc/NEWS index 6bde852f075..ddef4892216 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -233,6 +233,9 @@ Core and Builtins Library ------- +- Issue #16564: Fixed a performance regression relative to Python 3.1 in the + caching of compiled regular expressions. + - Issue #17431: Fix missing import of BytesFeedParser in email.parser. - Issue #1285086: Get rid of the refcounting hack and speed up From a99dfd1ab6c3aa9b5a45556e799989f42ac9b7f6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Mar 2013 22:59:27 +0200 Subject: [PATCH 5/5] Issue #16389: Fixed an issue number in previos commit. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index ddef4892216..c9c854dd2d5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -233,7 +233,7 @@ Core and Builtins Library ------- -- Issue #16564: Fixed a performance regression relative to Python 3.1 in the +- Issue #16389: Fixed a performance regression relative to Python 3.1 in the caching of compiled regular expressions. - Issue #17431: Fix missing import of BytesFeedParser in email.parser.