From 28a691b7fdde1b8abafa4c4a5025e6bfa44f48b9 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 17 Apr 2010 00:19:56 +0000 Subject: [PATCH] PEP 3147 --- .bzrignore | 1 + .hgignore | 1 + Doc/c-api/import.rst | 17 + Doc/library/compileall.rst | 12 +- Doc/library/imp.rst | 37 +- Doc/library/py_compile.rst | 10 +- Doc/library/runpy.rst | 5 +- Include/import.h | 3 + Lib/compileall.py | 74 ++-- Lib/importlib/_bootstrap.py | 21 +- Lib/importlib/test/__main__.py | 7 +- Lib/importlib/test/source/test_file_loader.py | 9 +- Lib/importlib/test/source/test_finder.py | 13 +- .../test/source/test_source_encoding.py | 2 +- Lib/importlib/test/source/util.py | 19 +- Lib/importlib/util.py | 1 + Lib/inspect.py | 1 + Lib/py_compile.py | 48 ++- Lib/pydoc.py | 3 +- Lib/runpy.py | 2 + Lib/site.py | 12 +- Lib/test/script_helper.py | 23 +- Lib/test/support.py | 90 ++-- Lib/test/test_cmd_line_script.py | 30 +- Lib/test/test_compileall.py | 79 +++- Lib/test/test_frozen.py | 10 +- Lib/test/test_imp.py | 127 +++++- Lib/test/test_import.py | 206 +++++++-- Lib/test/test_pkg.py | 20 +- Lib/test/test_pkgimport.py | 26 +- Lib/test/test_pydoc.py | 16 +- Lib/test/test_runpy.py | 25 +- Lib/test/test_site.py | 45 +- Lib/test/test_zipfile.py | 9 +- Lib/test/test_zipimport.py | 39 +- Lib/zipfile.py | 53 ++- Makefile.pre.in | 1 + Python/import.c | 391 ++++++++++++++++-- Python/pythonrun.c | 2 + 39 files changed, 1203 insertions(+), 287 deletions(-) diff --git a/.bzrignore b/.bzrignore index e5885958ba0..d2ba64da335 100644 --- a/.bzrignore +++ b/.bzrignore @@ -33,3 +33,4 @@ Parser/pgen Lib/test/data/* Lib/lib2to3/Grammar*.pickle Lib/lib2to3/PatternGrammar*.pickle +__pycache__ diff --git a/.hgignore b/.hgignore index 3ed3aefb665..e02d110a57f 100644 --- a/.hgignore +++ b/.hgignore @@ -54,3 +54,4 @@ PCbuild/*.o PCbuild/*.ncb PCbuild/*.bsc PCbuild/Win32-temp-* +__pycache__ diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 71c9d839120..de03f6e173f 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -124,12 +124,24 @@ Importing Modules If *name* points to a dotted name of the form ``package.module``, any package structures not already created will still not be created. + See also :func:`PyImport_ExecCodeModuleEx` and + :func:`PyImport_ExecCodeModuleWithPathnames`. + .. cfunction:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) Like :cfunc:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of the module object is set to *pathname* if it is non-``NULL``. + See also :func:`PyImport_ExecCodeModuleWithPathnames`. + + +.. cfunction:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) + + Like :cfunc:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + attribute of the module object is set to *cpathname* if it is + non-``NULL``. Of the three functions, this is the preferred one to use. + .. cfunction:: long PyImport_GetMagicNumber() @@ -138,6 +150,11 @@ Importing Modules of the bytecode file, in little-endian byte order. +.. cfunction:: const char * PyImport_GetMagicTag() + + Return the magic tag string for :pep:`3147` format Python bytecode file + names. + .. cfunction:: PyObject* PyImport_GetModuleDict() Return the dictionary used for the module administration (a.k.a. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 83e418d30c5..b0a2e34d911 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -17,9 +17,11 @@ line. If no arguments are given, the invocation is equivalent to ``-l sys.path``. Printing lists of the files compiled can be disabled with the :option:`-q` flag. In addition, the :option:`-x` option takes a regular expression argument. All files that match the expression will be skipped. +The :option:`-b` flag may be given to write legacy ``.pyc`` file path names, +otherwise :pep:`3147` style byte-compiled path names are written. -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. The *maxlevels* parameter is used to limit the depth of @@ -34,12 +36,16 @@ expression argument. All files that match the expression will be skipped. If *quiet* is true, nothing is printed to the standard output in normal operation. + If *legacy* is true, old-style ``.pyc`` file path names are written, + otherwise (the default), :pep:`3147` style path names are written. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False) + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False) Byte-compile all the :file:`.py` files found along ``sys.path``. If *skip_curdir* is true (the default), the current directory is not included in - the search. The *maxlevels* and *force* parameters default to ``0`` and are + the search. The *maxlevels* parameter defaults to ``0``, and the *force* + and *legacy* parameters default to ``False``. All are passed to the :func:`compile_dir` function. To force a recompile of all the :file:`.py` files in the :file:`Lib/` diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 68cda84bf5c..6e70d084fba 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -204,8 +204,41 @@ This module provides an interface to the mechanisms used to implement the function does nothing. -The following constants with integer values, defined in this module, are used to -indicate the search result of :func:`find_module`. +The following functions and data provide conveniences for handling :pep:`3147` +byte-compiled file paths. + +.. versionadded:: 3.2 + +.. function:: cache_from_source(path, debug_override=None) + + Return the PEP 3147 path to the byte-compiled file associated with the + source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return + value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. + The ``cpython-32`` string comes from the current magic tag (see + :func:`get_tag`). The returned path will end in ``.pyc`` when + ``__debug__`` is True or ``.pyo`` for an optimized Python + (i.e. ``__debug__`` is False). By passing in True or False for + *debug_override* you can override the system's value for ``__debug__`` for + extension selection. + + *path* need not exist. + +.. function:: source_from_cache(path) + + Given the *path* to a PEP 3147 file name, return the associated source code + file path. For example, if *path* is + ``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be + ``/foo/bar/baz.py``. *path* need not exist, however if it does not conform + to PEP 3147 format, a ``ValueError`` is raised. + +.. function:: get_tag() + + Return the PEP 3147 magic tag string matching this version of Python's + magic number, as returned by :func:`get_magic`. + + +The following constants with integer values, defined in this module, are used +to indicate the search result of :func:`find_module`. .. data:: PY_SOURCE diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index c4f72297dae..c6eea8499a1 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -26,12 +26,16 @@ byte-code cache files in the directory containing the source code. Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file name *file*. The byte-code is written to - *cfile*, which defaults to *file* ``+`` ``'c'`` (``'o'`` if optimization is - enabled in the current interpreter). If *dfile* is specified, it is used as the + *cfile*, which defaults to the :PEP:`3147` path, ending in ``.pyc`` + (``'.pyo`` if optimization is enabled in the current interpreter). For + example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to + ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is specified, it is used as the name of the source file in error messages instead of *file*. If *doraise* is true, a :exc:`PyCompileError` is raised when an error is encountered while compiling *file*. If *doraise* is false (the default), an error string is - written to ``sys.stderr``, but no exception is raised. + written to ``sys.stderr``, but no exception is raised. This function + returns the path to byte-compiled file, i.e. whatever *cfile* value was + used. .. function:: main(args=None) diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 7278b7a0dfd..a96285cf7d6 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -32,7 +32,8 @@ The :mod:`runpy` module provides two functions: below are defined in the supplied dictionary, those definitions are overridden by :func:`run_module`. - The special global variables ``__name__``, ``__file__``, ``__loader__`` + The special global variables ``__name__``, ``__file__``, ``__cached__``, + ``__loader__`` and ``__package__`` are set in the globals dictionary before the module code is executed (Note that this is a minimal set of variables - other variables may be set implicitly as an interpreter implementation detail). @@ -45,6 +46,8 @@ The :mod:`runpy` module provides two functions: loader does not make filename information available, this variable is set to :const:`None`. + ``__cached__`` will be set to ``None``. + ``__loader__`` is set to the PEP 302 module loader used to retrieve the code for the module (This loader may be a wrapper around the standard import mechanism). diff --git a/Include/import.h b/Include/import.h index b8de2fd90b4..923fbca4d8c 100644 --- a/Include/import.h +++ b/Include/import.h @@ -8,9 +8,12 @@ extern "C" { #endif PyAPI_FUNC(long) PyImport_GetMagicNumber(void); +PyAPI_FUNC(const char *) PyImport_GetMagicTag(void); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(char *name, PyObject *co); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx( char *name, PyObject *co, char *pathname); +PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames( + char *name, PyObject *co, char *pathname, char *cpathname); PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); PyAPI_FUNC(PyObject *) PyImport_AddModule(const char *name); PyAPI_FUNC(PyObject *) PyImport_ImportModule(const char *name); diff --git a/Lib/compileall.py b/Lib/compileall.py index ae86292790f..d9d78161cc3 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -12,6 +12,7 @@ See module py_compile for details of the actual byte-compilation. """ import os +import errno import sys import py_compile import struct @@ -20,7 +21,7 @@ import imp __all__ = ["compile_dir","compile_file","compile_path"] def compile_dir(dir, maxlevels=10, ddir=None, - force=0, rx=None, quiet=0): + force=False, rx=None, quiet=False, legacy=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -29,8 +30,9 @@ def compile_dir(dir, maxlevels=10, ddir=None, maxlevels: maximum recursion level (default 10) ddir: if given, purported directory name (this is the directory name that will show up in error messages) - force: if 1, force compilation, even if timestamps are up-to-date - quiet: if 1, be quiet during compilation + force: if True, force compilation, even if timestamps are up-to-date + quiet: if True, be quiet during compilation + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths """ if not quiet: @@ -49,24 +51,26 @@ def compile_dir(dir, maxlevels=10, ddir=None, else: dfile = None if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet): + if not compile_file(fullname, ddir, force, rx, quiet, legacy): success = 0 elif maxlevels > 0 and \ name != os.curdir and name != os.pardir and \ os.path.isdir(fullname) and \ not os.path.islink(fullname): if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet): + quiet, legacy): success = 0 return success -def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): +def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, + legacy=False): """Byte-compile file. - file: the file to byte-compile + fullname: the file to byte-compile ddir: if given, purported directory name (this is the directory name that will show up in error messages) - force: if 1, force compilation, even if timestamps are up-to-date - quiet: if 1, be quiet during compilation + force: if True, force compilation, even if timestamps are up-to-date + quiet: if True, be quiet during compilation + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths """ success = 1 @@ -80,13 +84,22 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): if mo: return success if os.path.isfile(fullname): + if legacy: + cfile = fullname + ('c' if __debug__ else 'o') + else: + cfile = imp.cache_from_source(fullname) + cache_dir = os.path.dirname(cfile) + try: + os.mkdir(cache_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise head, tail = name[:-3], name[-3:] if tail == '.py': if not force: try: mtime = int(os.stat(fullname).st_mtime) expect = struct.pack('<4sl', imp.get_magic(), mtime) - cfile = fullname + (__debug__ and 'c' or 'o') with open(cfile, 'rb') as chandle: actual = chandle.read(8) if expect == actual: @@ -96,14 +109,15 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): if not quiet: print('Compiling', fullname, '...') try: - ok = py_compile.compile(fullname, None, dfile, True) + ok = py_compile.compile(fullname, cfile, dfile, True) except py_compile.PyCompileError as err: if quiet: print('*** Error compiling', fullname, '...') else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace') + msg = err.msg.encode(sys.stdout.encoding, + errors='backslashreplace') msg = msg.decode(sys.stdout.encoding) print(msg) success = 0 @@ -119,15 +133,17 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): success = 0 return success -def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0): +def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, + legacy=False): """Byte-compile all module on sys.path. Arguments (all optional): skip_curdir: if true, skip current directory (default true) maxlevels: max recursion level (default 0) - force: as for compile_dir() (default 0) - quiet: as for compile_dir() (default 0) + force: as for compile_dir() (default False) + quiet: as for compile_dir() (default False) + legacy: as for compile_dir() (default False) """ success = 1 @@ -136,7 +152,8 @@ def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0): print('Skipping current directory') else: success = success and compile_dir(dir, maxlevels, None, - force, quiet=quiet) + force, quiet=quiet, + legacy=legacy) return success def expand_args(args, flist): @@ -162,10 +179,10 @@ def main(): """Script main program.""" import getopt try: - opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:') + opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:b') except getopt.error as msg: print(msg) - print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] " \ + print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] " "[-x regexp] [-i list] [directory|file ...]") print("-l: don't recurse down") print("-f: force rebuild even if timestamps are up-to-date") @@ -174,23 +191,27 @@ def main(): print(" if no directory arguments, -l sys.path is assumed") print("-x regexp: skip files matching the regular expression regexp") print(" the regexp is searched for in the full path of the file") - print("-i list: expand list with its content (file and directory names)") + print("-i list: expand list with its content " + "(file and directory names)") + print("-b: Produce legacy byte-compile file paths") sys.exit(2) maxlevels = 10 ddir = None - force = 0 - quiet = 0 + force = False + quiet = False rx = None flist = None + legacy = False for o, a in opts: if o == '-l': maxlevels = 0 if o == '-d': ddir = a - if o == '-f': force = 1 - if o == '-q': quiet = 1 + if o == '-f': force = True + if o == '-q': quiet = True if o == '-x': import re rx = re.compile(a) if o == '-i': flist = a + if o == '-b': legacy = True if ddir: if len(args) != 1 and not os.path.isdir(args[0]): print("-d destdir require exactly one directory argument") @@ -207,13 +228,14 @@ def main(): for arg in args: if os.path.isdir(arg): if not compile_dir(arg, maxlevels, ddir, - force, rx, quiet): + force, rx, quiet, legacy): success = 0 else: - if not compile_file(arg, ddir, force, rx, quiet): + if not compile_file(arg, ddir, force, rx, + quiet, legacy): success = 0 else: - success = compile_path() + success = compile_path(legacy=legacy) except KeyboardInterrupt: print("\n[interrupt]") success = 0 diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 330eb6336c3..30d5251f3ac 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -488,6 +488,16 @@ class _PyPycFileLoader(PyPycLoader, _PyFileLoader): """Load a module from a source or bytecode file.""" + def _find_path(self, ext_type): + """Return PEP 3147 path if ext_type is PY_COMPILED, otherwise + super()._find_path() is called.""" + if ext_type == imp.PY_COMPILED: + # We don't really care what the extension on self._base_path is, + # as long as it has exactly one dot. + bytecode_path = imp.cache_from_source(self._base_path + '.py') + return (bytecode_path if _path_exists(bytecode_path) else None) + return super()._find_path(ext_type) + @_check_name def source_mtime(self, name): """Return the modification time of the source for the specified @@ -515,7 +525,16 @@ class _PyPycFileLoader(PyPycLoader, _PyFileLoader): """ bytecode_path = self.bytecode_path(name) if not bytecode_path: - bytecode_path = self._base_path + _suffix_list(imp.PY_COMPILED)[0] + source_path = self.source_path(name) + bytecode_path = imp.cache_from_source(source_path) + # Ensure that the __pycache__ directory exists. We can't use + # os.path.dirname() here. + dirname, sep, basename = bytecode_path.rpartition(path_sep) + try: + _os.mkdir(dirname) + except OSError as error: + if error.errno != errno.EEXIST: + raise try: # Assuming bytes. with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file: diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py index b97e3821959..83292641744 100644 --- a/Lib/importlib/test/__main__.py +++ b/Lib/importlib/test/__main__.py @@ -13,7 +13,12 @@ import unittest def test_main(): - start_dir = os.path.dirname(__file__) + if '__pycache__' in __file__: + parts = __file__.split(os.path.sep) + start_dir = sep.join(parts[:-2]) + else: + start_dir = os.path.dirname(__file__) + # XXX 2010-03-18 barry: Fix __file__ top_dir = os.path.dirname(os.path.dirname(start_dir)) test_loader = unittest.TestLoader() if '--builtin' in sys.argv: diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index ae4b185a359..905940501c6 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -127,7 +127,7 @@ class BadBytecodeTest(unittest.TestCase): except KeyError: pass py_compile.compile(mapping[name]) - bytecode_path = source_util.bytecode_path(mapping[name]) + bytecode_path = imp.cache_from_source(mapping[name]) with open(bytecode_path, 'rb') as file: bc = file.read() new_bc = manipulator(bc) @@ -226,7 +226,7 @@ class BadBytecodeTest(unittest.TestCase): zeros = b'\x00\x00\x00\x00' with source_util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(4) bytecode_file.write(zeros) @@ -242,9 +242,10 @@ class BadBytecodeTest(unittest.TestCase): def test_bad_marshal(self): # Bad marshal data should raise a ValueError. with source_util.create_modules('_temp') as mapping: - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) source_mtime = os.path.getmtime(mapping['_temp']) source_timestamp = importlib._w_long(source_mtime) + source_util.ensure_bytecode_path(bytecode_path) with open(bytecode_path, 'wb') as bytecode_file: bytecode_file.write(imp.get_magic()) bytecode_file.write(source_timestamp) @@ -260,7 +261,7 @@ class BadBytecodeTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index 8f15f6245b1..16736694751 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,9 @@ from importlib import _bootstrap from .. import abc from . import util as source_util +from test.support import make_legacy_pyc import os +import errno import py_compile import unittest import warnings @@ -52,6 +54,14 @@ class FinderTests(abc.FinderTests): if unlink: for name in unlink: os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise loader = self.import_(mapping['.root'], test) self.assertTrue(hasattr(loader, 'load_module')) return loader @@ -60,7 +70,8 @@ class FinderTests(abc.FinderTests): # [top-level source] self.run_test('top_level') # [top-level bc] - self.run_test('top_level', compile_={'top_level'}, unlink={'top_level'}) + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) # [top-level both] self.run_test('top_level', compile_={'top_level'}) diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/importlib/test/source/test_source_encoding.py index fde355f3af6..04aac249410 100644 --- a/Lib/importlib/test/source/test_source_encoding.py +++ b/Lib/importlib/test/source/test_source_encoding.py @@ -33,7 +33,7 @@ class EncodingTest(unittest.TestCase): def run_test(self, source): with source_util.create_modules(self.module_name) as mapping: - with open(mapping[self.module_name], 'wb')as file: + with open(mapping[self.module_name], 'wb') as file: file.write(source) loader = _bootstrap._PyPycFileLoader(self.module_name, mapping[self.module_name], False) diff --git a/Lib/importlib/test/source/util.py b/Lib/importlib/test/source/util.py index 2b945c59364..ae65663a670 100644 --- a/Lib/importlib/test/source/util.py +++ b/Lib/importlib/test/source/util.py @@ -1,5 +1,6 @@ from .. import util import contextlib +import errno import functools import imp import os @@ -26,14 +27,16 @@ def writes_bytecode_files(fxn): return wrapper -def bytecode_path(source_path): - for suffix, _, type_ in imp.get_suffixes(): - if type_ == imp.PY_COMPILED: - bc_suffix = suffix - break - else: - raise ValueError("no bytecode suffix is defined") - return os.path.splitext(source_path)[0] + bc_suffix +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise @contextlib.contextmanager diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3abc6a957b8..7b44fa1344c 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,4 +1,5 @@ """Utility code for constructing importers, etc.""" + from ._bootstrap import module_for_loader from ._bootstrap import set_loader from ._bootstrap import set_package diff --git a/Lib/inspect.py b/Lib/inspect.py index b9fcd747840..ea30466433e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -54,6 +54,7 @@ def ismodule(object): """Return true if the object is a module. Module objects provide these attributes: + __cached__ pathname to byte compiled file __doc__ documentation string __file__ filename (missing for built-in modules)""" return isinstance(object, types.ModuleType) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 93618758463..f257770cbc0 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -4,6 +4,7 @@ This module has intimate knowledge of the format of .pyc files. """ import builtins +import errno import imp import marshal import os @@ -37,16 +38,18 @@ class PyCompileError(Exception): can be accesses as class variable 'file' msg: string message to be written as error message - If no value is given, a default exception message will be given, - consistent with 'standard' py_compile output. - message (or default) can be accesses as class variable 'msg' + If no value is given, a default exception message will be + given, consistent with 'standard' py_compile output. + message (or default) can be accesses as class variable + 'msg' """ def __init__(self, exc_type, exc_value, file, msg=''): exc_type_name = exc_type.__name__ if exc_type is SyntaxError: - tbtext = ''.join(traceback.format_exception_only(exc_type, exc_value)) + tbtext = ''.join(traceback.format_exception_only( + exc_type, exc_value)) errmsg = tbtext.replace('File ""', 'File "%s"' % file) else: errmsg = "Sorry: %s: %s" % (exc_type_name,exc_value) @@ -64,7 +67,7 @@ class PyCompileError(Exception): def wr_long(f, x): """Internal; write a 32-bit int to a file in little-endian order.""" - f.write(bytes([x & 0xff, + f.write(bytes([x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, (x >> 24) & 0xff])) @@ -72,20 +75,18 @@ def wr_long(f, x): def compile(file, cfile=None, dfile=None, doraise=False): """Byte-compile one Python source file to Python bytecode. - Arguments: - - file: source filename - cfile: target filename; defaults to source with 'c' or 'o' appended - ('c' normally, 'o' in optimizing mode, giving .pyc or .pyo) - dfile: purported filename; defaults to source (this is the filename - that will show up in error messages) - doraise: flag indicating whether or not an exception should be - raised when a compile error is found. If an exception - occurs and this flag is set to False, a string - indicating the nature of the exception will be printed, - and the function will return to the caller. If an - exception occurs and this flag is set to True, a - PyCompileError exception will be raised. + :param file: The source file name. + :param cfile: The target byte compiled file name. When not given, this + defaults to the PEP 3147 location. + :param dfile: Purported file name, i.e. the file name that shows up in + error messages. Defaults to the source file name. + :param doraise: Flag indicating whether or not an exception should be + raised when a compile error is found. If an exception occurs and this + flag is set to False, a string indicating the nature of the exception + will be printed, and the function will return to the caller. If an + exception occurs and this flag is set to True, a PyCompileError + exception will be raised. + :return: Path to the resulting byte compiled file. Note that it isn't necessary to byte-compile Python modules for execution efficiency -- Python itself byte-compiles a module when @@ -102,7 +103,6 @@ def compile(file, cfile=None, dfile=None, doraise=False): See compileall.py for a script/module that uses this module to byte-compile all installed files (or all files in selected directories). - """ with open(file, "rb") as f: encoding = tokenize.detect_encoding(f.readline)[0] @@ -122,7 +122,12 @@ def compile(file, cfile=None, dfile=None, doraise=False): sys.stderr.write(py_exc.msg + '\n') return if cfile is None: - cfile = file + (__debug__ and 'c' or 'o') + cfile = imp.cache_from_source(file) + try: + os.mkdir(os.path.dirname(cfile)) + except OSError as error: + if error.errno != errno.EEXIST: + raise with open(cfile, 'wb') as fc: fc.write(b'\0\0\0\0') wr_long(fc, timestamp) @@ -130,6 +135,7 @@ def compile(file, cfile=None, dfile=None, doraise=False): fc.flush() fc.seek(0, 0) fc.write(MAGIC) + return cfile def main(args=None): """Compile several source files. diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 6b9dd3db2c7..208740fbbf5 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -159,7 +159,8 @@ def visiblename(name, all=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', - '__module__', '__name__', '__slots__', '__package__') + '__module__', '__name__', '__slots__', '__package__', + '__cached__') if name in _hidden_names: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 diff --git a/Lib/runpy.py b/Lib/runpy.py index 6e94d6bd752..f25108186c2 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -67,6 +67,7 @@ def _run_code(code, run_globals, init_globals=None, run_globals.update(init_globals) run_globals.update(__name__ = mod_name, __file__ = mod_fname, + __cached__ = None, __loader__ = mod_loader, __package__ = pkg_name) exec(code, run_globals) @@ -130,6 +131,7 @@ def _run_module_as_main(mod_name, alter_argv=True): At the very least, these variables in __main__ will be overwritten: __name__ __file__ + __cached__ __loader__ __package__ """ diff --git a/Lib/site.py b/Lib/site.py index 55e662c0b0f..d99b538b5c4 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -74,15 +74,19 @@ def makepath(*paths): return dir, os.path.normcase(dir) -def abs__file__(): - """Set all module' __file__ attribute to an absolute path""" +def abs_paths(): + """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): if hasattr(m, '__loader__'): continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) except AttributeError: - continue + pass + try: + m.__cached__ = os.path.abspath(m.__cached__) + except AttributeError: + pass def removeduppaths(): @@ -518,7 +522,7 @@ def execusercustomize(): def main(): global ENABLE_USER_SITE - abs__file__() + abs_paths() known_paths = removeduppaths() if (os.name == "posix" and sys.path and os.path.basename(sys.path[-1]) == "Modules"): diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py index 144cf660e87..39874d97ece 100644 --- a/Lib/test/script_helper.py +++ b/Lib/test/script_helper.py @@ -11,6 +11,9 @@ import contextlib import shutil import zipfile +from imp import source_from_cache +from test.support import make_legacy_pyc + # Executing the interpreter in a subprocess def python_exit_code(*args): cmd_line = [sys.executable, '-E'] @@ -62,20 +65,18 @@ def make_script(script_dir, script_basename, source): script_file.close() return script_name -def compile_script(script_name): - py_compile.compile(script_name, doraise=True) - if __debug__: - compiled_name = script_name + 'c' - else: - compiled_name = script_name + 'o' - return compiled_name - def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') if name_in_zip is None: - name_in_zip = os.path.basename(script_name) + parts = script_name.split(os.sep) + if len(parts) >= 2 and parts[-2] == '__pycache__': + legacy_pyc = make_legacy_pyc(source_from_cache(script_name)) + name_in_zip = os.path.basename(legacy_pyc) + script_name = legacy_pyc + else: + name_in_zip = os.path.basename(script_name) zip_file.write(script_name, name_in_zip) zip_file.close() #if test.test_support.verbose: @@ -98,8 +99,8 @@ def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, script_name = make_script(zip_dir, script_basename, source) unlink.append(script_name) if compiled: - init_name = compile_script(init_name) - script_name = compile_script(script_name) + init_name = py_compile(init_name, doraise=True) + script_name = py_compile(script_name, doraise=True) unlink.extend((init_name, script_name)) pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)] script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name)) diff --git a/Lib/test/support.py b/Lib/test/support.py index 9f9292d975f..3c0002b0d02 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -17,22 +17,25 @@ import unittest import importlib import collections import re +import imp import time -__all__ = ["Error", "TestFailed", "ResourceDenied", "import_module", - "verbose", "use_resources", "max_memuse", "record_original_stdout", - "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "find_unused_port", "bind_port", - "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", - "findfile", "sortdict", "check_syntax_error", "open_urlresource", - "check_warnings", "CleanImport", "EnvironmentVarGuard", - "TransientResource", "captured_output", "captured_stdout", - "time_out", "socket_peer_reset", "ioerror_peer_reset", - "run_with_locale", - "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", - "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", - "reap_children", "cpython_only", "check_impl_detail", "get_attribute", - "swap_item", "swap_attr"] +__all__ = [ + "Error", "TestFailed", "ResourceDenied", "import_module", + "verbose", "use_resources", "max_memuse", "record_original_stdout", + "get_original_stdout", "unload", "unlink", "rmtree", "forget", + "is_resource_enabled", "requires", "find_unused_port", "bind_port", + "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", + "findfile", "sortdict", "check_syntax_error", "open_urlresource", + "check_warnings", "CleanImport", "EnvironmentVarGuard", + "TransientResource", "captured_output", "captured_stdout", + "time_out", "socket_peer_reset", "ioerror_peer_reset", + "run_with_locale", 'temp_umask', + "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", + "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", + "reap_children", "cpython_only", "check_impl_detail", "get_attribute", + "swap_item", "swap_attr", + ] class Error(Exception): @@ -177,27 +180,50 @@ def unload(name): def unlink(filename): try: os.unlink(filename) - except OSError: - pass + except OSError as error: + # The filename need not exist. + if error.errno != errno.ENOENT: + raise def rmtree(path): try: shutil.rmtree(path) - except OSError as e: + except OSError as error: # Unix returns ENOENT, Windows returns ESRCH. - if e.errno not in (errno.ENOENT, errno.ESRCH): + if error.errno not in (errno.ENOENT, errno.ESRCH): raise +def make_legacy_pyc(source): + """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. + + The choice of .pyc or .pyo extension is done based on the __debug__ flag + value. + + :param source: The file system path to the source file. The source file + does not need to exist, however the PEP 3147 pyc file must exist. + :return: The file system path to the legacy pyc file. + """ + pyc_file = imp.cache_from_source(source) + up_one = os.path.dirname(os.path.abspath(source)) + legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o')) + os.rename(pyc_file, legacy_pyc) + return legacy_pyc + def forget(modname): - '''"Forget" a module was ever imported by removing it from sys.modules and - deleting any .pyc and .pyo files.''' + """'Forget' a module was ever imported. + + This removes the module from sys.modules and deletes any PEP 3147 or + legacy .pyc and .pyo files. + """ unload(modname) for dirname in sys.path: - unlink(os.path.join(dirname, modname + '.pyc')) - # Deleting the .pyo file cannot be within the 'try' for the .pyc since - # the chance exists that there is no .pyc (and thus the 'try' statement - # is exited) but there is a .pyo file. - unlink(os.path.join(dirname, modname + '.pyo')) + source = os.path.join(dirname, modname + '.py') + # It doesn't matter if they exist or not, unlink all possible + # combinations of PEP 3147 and legacy pyc and pyo files. + unlink(source + 'c') + unlink(source + 'o') + unlink(imp.cache_from_source(source, debug_override=True)) + unlink(imp.cache_from_source(source, debug_override=False)) def is_resource_enabled(resource): """Test whether a resource is enabled. Known resources are set by @@ -208,7 +234,9 @@ def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available. If the caller's module is __main__ then automatically return True. The - possibility of False being returned occurs when regrtest.py is executing.""" + possibility of False being returned occurs when regrtest.py is + executing. + """ # see if the caller's module is __main__ - if so, treat as if # the resource was set if sys._getframe(1).f_globals.get("__name__") == "__main__": @@ -405,6 +433,16 @@ def temp_cwd(name='tempcwd', quiet=False): rmtree(name) +@contextlib.contextmanager +def temp_umask(umask): + """Context manager that temporarily sets the process umask.""" + oldmask = os.umask(umask) + try: + yield + finally: + os.umask(oldmask) + + def findfile(file, here=__file__, subdir=None): """Try to find a file on sys.path and the working directory. If it is not found the argument passed to the function is returned (this does not diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f7c27a73158..3f4dd6d9193 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -1,12 +1,14 @@ -# Tests command line execution of scripts +# tests command line execution of scripts import unittest import os import os.path +import py_compile + import test.support -from test.script_helper import (run_python, - temp_dir, make_script, compile_script, - make_pkg, make_zip_script, make_zip_pkg) +from test.script_helper import ( + make_pkg, make_script, make_zip_pkg, make_zip_script, run_python, + temp_dir) verbose = test.support.verbose @@ -28,6 +30,7 @@ assertEqual(result, ['Top level assignment', 'Lower level reference']) # Check population of magic variables assertEqual(__name__, '__main__') print('__file__==%r' % __file__) +assertEqual(__cached__, None) print('__package__==%r' % __package__) # Check the sys module import sys @@ -101,9 +104,10 @@ class CmdLineTest(unittest.TestCase): def test_script_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(compiled_name, compiled_name, compiled_name, None) + self._check_script(compiled_name, compiled_name, + compiled_name, None) def test_directory(self): with temp_dir() as script_dir: @@ -113,9 +117,10 @@ class CmdLineTest(unittest.TestCase): def test_directory_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(script_dir, compiled_name, script_dir, '') + pyc_file = test.support.make_legacy_pyc(script_name) + self._check_script(script_dir, pyc_file, script_dir, '') def test_directory_error(self): with temp_dir() as script_dir: @@ -131,7 +136,7 @@ class CmdLineTest(unittest.TestCase): def test_zipfile_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) self._check_script(zip_name, run_name, zip_name, '') @@ -176,11 +181,12 @@ class CmdLineTest(unittest.TestCase): pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) + pyc_file = test.support.make_legacy_pyc(script_name) launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, compiled_name, - compiled_name, 'test_pkg') + self._check_script(launch_name, pyc_file, + pyc_file, 'test_pkg') def test_package_error(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 4b6feba24ee..8b345873906 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -5,22 +5,23 @@ import os import py_compile import shutil import struct +import subprocess import tempfile -from test import support import unittest import io +from test import support class CompileallTests(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.bc_path = self.source_path + ('c' if __debug__ else 'o') + self.bc_path = imp.cache_from_source(self.source_path) with open(self.source_path, 'w') as file: file.write('x = 123\n') self.source_path2 = os.path.join(self.directory, '_test2.py') - self.bc_path2 = self.source_path2 + ('c' if __debug__ else 'o') + self.bc_path2 = imp.cache_from_source(self.source_path2) shutil.copyfile(self.source_path, self.source_path2) def tearDown(self): @@ -65,17 +66,19 @@ class CompileallTests(unittest.TestCase): except: pass compileall.compile_file(self.source_path, force=False, quiet=True) - self.assertTrue(os.path.isfile(self.bc_path) \ - and not os.path.isfile(self.bc_path2)) + self.assertTrue(os.path.isfile(self.bc_path) and + not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) compileall.compile_dir(self.directory, force=False, quiet=True) - self.assertTrue(os.path.isfile(self.bc_path) \ - and os.path.isfile(self.bc_path2)) + self.assertTrue(os.path.isfile(self.bc_path) and + os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + class EncodingTest(unittest.TestCase): - 'Issue 6716: compileall should escape source code when printing errors to stdout.' + """Issue 6716: compileall should escape source code when printing errors + to stdout.""" def setUp(self): self.directory = tempfile.mkdtemp() @@ -95,9 +98,65 @@ class EncodingTest(unittest.TestCase): finally: sys.stdout = orig_stdout +class CommandLineTests(unittest.TestCase): + """Test some aspects of compileall's CLI.""" + + def setUp(self): + self.addCleanup(self._cleanup) + self.directory = tempfile.mkdtemp() + self.pkgdir = os.path.join(self.directory, 'foo') + os.mkdir(self.pkgdir) + # Touch the __init__.py and a package module. + with open(os.path.join(self.pkgdir, '__init__.py'), 'w'): + pass + with open(os.path.join(self.pkgdir, 'bar.py'), 'w'): + pass + sys.path.insert(0, self.directory) + + def _cleanup(self): + support.rmtree(self.directory) + assert sys.path[0] == self.directory, 'Missing path' + del sys.path[0] + + def test_pep3147_paths(self): + # Ensure that the default behavior of compileall's CLI is to create + # PEP 3147 pyc/pyo files. + retcode = subprocess.call( + (sys.executable, '-m', 'compileall', '-q', self.pkgdir)) + self.assertEqual(retcode, 0) + # Verify the __pycache__ directory contents. + cachedir = os.path.join(self.pkgdir, '__pycache__') + self.assertTrue(os.path.exists(cachedir)) + ext = ('pyc' if __debug__ else 'pyo') + expected = sorted(base.format(imp.get_tag(), ext) for base in + ('__init__.{}.{}', 'bar.{}.{}')) + self.assertEqual(sorted(os.listdir(cachedir)), expected) + # Make sure there are no .pyc files in the source directory. + self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir) + if pyc_file.endswith(ext)]) + + def test_legacy_paths(self): + # Ensure that with the proper switch, compileall leaves legacy + # pyc/pyo files, and no __pycache__ directory. + retcode = subprocess.call( + (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir)) + self.assertEqual(retcode, 0) + # Verify the __pycache__ directory contents. + cachedir = os.path.join(self.pkgdir, '__pycache__') + self.assertFalse(os.path.exists(cachedir)) + ext = ('pyc' if __debug__ else 'pyo') + expected = [base.format(ext) for base in ('__init__.{}', 'bar.{}')] + expected.extend(['__init__.py', 'bar.py']) + expected.sort() + self.assertEqual(sorted(os.listdir(self.pkgdir)), expected) + + def test_main(): - support.run_unittest(CompileallTests, - EncodingTest) + support.run_unittest( + CommandLineTests, + CompileallTests, + EncodingTest, + ) if __name__ == "__main__": diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py index 79cc1c340d8..28186bbd4d1 100644 --- a/Lib/test/test_frozen.py +++ b/Lib/test/test_frozen.py @@ -11,7 +11,7 @@ class FrozenTests(unittest.TestCase): except ImportError as x: self.fail("import __hello__ failed:" + str(x)) self.assertEqual(__hello__.initialized, True) - self.assertEqual(len(dir(__hello__)), 6, dir(__hello__)) + self.assertEqual(len(dir(__hello__)), 7, dir(__hello__)) try: import __phello__ @@ -19,9 +19,9 @@ class FrozenTests(unittest.TestCase): self.fail("import __phello__ failed:" + str(x)) self.assertEqual(__phello__.initialized, True) if not "__phello__.spam" in sys.modules: - self.assertEqual(len(dir(__phello__)), 7, dir(__phello__)) - else: self.assertEqual(len(dir(__phello__)), 8, dir(__phello__)) + else: + self.assertEqual(len(dir(__phello__)), 9, dir(__phello__)) self.assertEquals(__phello__.__path__, [__phello__.__name__]) try: @@ -29,8 +29,8 @@ class FrozenTests(unittest.TestCase): except ImportError as x: self.fail("import __phello__.spam failed:" + str(x)) self.assertEqual(__phello__.spam.initialized, True) - self.assertEqual(len(dir(__phello__.spam)), 6) - self.assertEqual(len(dir(__phello__)), 8) + self.assertEqual(len(dir(__phello__.spam)), 7) + self.assertEqual(len(dir(__phello__)), 9) try: import __phello__.foo diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index e995bf06302..6412f3f9741 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,6 +1,7 @@ import imp import os import os.path +import shutil import sys import unittest from test import support @@ -139,7 +140,8 @@ class ImportTests(unittest.TestCase): mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') self.assertEqual(mod.a, 1) - mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc') + mod = imp.load_compiled( + temp_mod_name, imp.cache_from_source(temp_mod_name + '.py')) self.assertEqual(mod.a, 1) if not os.path.exists(test_package_name): @@ -184,11 +186,132 @@ class ReloadTests(unittest.TestCase): imp.reload(marshal) +class PEP3147Tests(unittest.TestCase): + """Tests of PEP 3147.""" + + tag = imp.get_tag() + + def test_cache_from_source(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyc file (i.e. under __pycache__). + self.assertEqual( + imp.cache_from_source('/foo/bar/baz/qux.py', True), + '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)) + + def test_cache_from_source_optimized(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyo file (i.e. under __pycache__). + self.assertEqual( + imp.cache_from_source('/foo/bar/baz/qux.py', False), + '/foo/bar/baz/__pycache__/qux.{}.pyo'.format(self.tag)) + + def test_cache_from_source_cwd(self): + self.assertEqual(imp.cache_from_source('foo.py', True), + os.sep.join(('__pycache__', + 'foo.{}.pyc'.format(self.tag)))) + + def test_cache_from_source_override(self): + # When debug_override is not None, it can be any true-ish or false-ish + # value. + self.assertEqual( + imp.cache_from_source('/foo/bar/baz.py', []), + '/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag)) + self.assertEqual( + imp.cache_from_source('/foo/bar/baz.py', [17]), + '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag)) + # However if the bool-ishness can't be determined, the exception + # propagates. + class Bearish: + def __bool__(self): raise RuntimeError + self.assertRaises( + RuntimeError, + imp.cache_from_source, '/foo/bar/baz.py', Bearish()) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_altsep_cache_from_source(self): + # Windows path and PEP 3147. + self.assertEqual( + imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True), + '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where altsep is right of sep. + self.assertEqual( + imp.cache_from_source('\\foo\\bar/baz\\qux.py', True), + '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_sep_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where sep is right of altsep. + self.assertEqual( + imp.cache_from_source('\\foo\\bar\\baz/qux.py', True), + '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag)) + + def test_source_from_cache(self): + # Given the path to a PEP 3147 defined .pyc file, return the path to + # its source. This tests the good path. + self.assertEqual(imp.source_from_cache( + '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)), + '/foo/bar/baz/qux.py') + + def test_source_from_cache_bad_path(self): + # When the path to a pyc file is not in PEP 3147 format, a ValueError + # is raised. + self.assertRaises( + ValueError, imp.source_from_cache, '/foo/bar/bazqux.pyc') + + def test_source_from_cache_no_slash(self): + # No slashes at all in path -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, 'foo.cpython-32.pyc') + + def test_source_from_cache_too_few_dots(self): + # Too few dots in final path component -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, '__pycache__/foo.pyc') + + def test_source_from_cache_too_many_dots(self): + # Too many dots in final path component -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, + '__pycache__/foo.cpython-32.foo.pyc') + + def test_source_from_cache_no__pycache__(self): + # Another problem with the path -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, + '/foo/bar/foo.cpython-32.foo.pyc') + + def test_package___file__(self): + # Test that a package's __file__ points to the right source directory. + os.mkdir('pep3147') + sys.path.insert(0, os.curdir) + def cleanup(): + if sys.path[0] == os.curdir: + del sys.path[0] + shutil.rmtree('pep3147') + self.addCleanup(cleanup) + # Touch the __init__.py file. + with open('pep3147/__init__.py', 'w'): + pass + m = __import__('pep3147') + # Ensure we load the pyc file. + support.forget('pep3147') + m = __import__('pep3147') + self.assertEqual(m.__file__, + os.sep.join(('.', 'pep3147', '__init__.py'))) + + def test_main(): tests = [ ImportTests, + PEP3147Tests, ReloadTests, - ] + ] try: import _thread except ImportError: diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index 9b34467ab09..0a21e182eb7 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -1,4 +1,5 @@ import builtins +import errno import imp import marshal import os @@ -8,8 +9,11 @@ import shutil import stat import sys import unittest -from test.support import (unlink, TESTFN, unload, run_unittest, is_jython, - check_warnings, EnvironmentVarGuard, swap_attr, swap_item) + +from test.support import ( + EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, + make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, + unlink, unload) def remove_files(name): @@ -19,12 +23,18 @@ def remove_files(name): name + ".pyw", name + "$py.class"): unlink(f) + try: + shutil.rmtree('__pycache__') + except OSError as error: + if error.errno != errno.ENOENT: + raise class ImportTests(unittest.TestCase): def tearDown(self): unload(TESTFN) + setUp = tearDown def test_case_sensitivity(self): @@ -53,8 +63,8 @@ class ImportTests(unittest.TestCase): pyc = TESTFN + ".pyc" with open(source, "w") as f: - print("# This tests Python's ability to import a", ext, "file.", - file=f) + print("# This tests Python's ability to import a", + ext, "file.", file=f) a = random.randrange(1000) b = random.randrange(1000) print("a =", a, file=f) @@ -73,10 +83,10 @@ class ImportTests(unittest.TestCase): self.assertEqual(mod.b, b, "module loaded (%s) but contents invalid" % mod) finally: + forget(TESTFN) unlink(source) unlink(pyc) unlink(pyo) - unload(TESTFN) sys.path.insert(0, os.curdir) try: @@ -87,32 +97,31 @@ class ImportTests(unittest.TestCase): finally: del sys.path[0] - @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") + @unittest.skipUnless(os.name == 'posix', + "test meaningful only on posix systems") def test_execute_bit_not_copied(self): # Issue 6070: under posix .pyc files got their execute bit set if # the .py file had the execute bit set, but they aren't executable. - oldmask = os.umask(0o022) - sys.path.insert(0, os.curdir) - try: - fname = TESTFN + os.extsep + "py" - f = open(fname, 'w').close() - os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) - __import__(TESTFN) - fn = fname + 'c' - if not os.path.exists(fn): - fn = fname + 'o' + with temp_umask(0o022): + sys.path.insert(0, os.curdir) + try: + fname = TESTFN + os.extsep + "py" + f = open(fname, 'w').close() + os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) + __import__(TESTFN) + fn = imp.cache_from_source(fname) if not os.path.exists(fn): self.fail("__import__ did not result in creation of " "either a .pyc or .pyo file") - s = os.stat(fn) - self.assertEqual(stat.S_IMODE(s.st_mode), - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - finally: - os.umask(oldmask) - remove_files(TESTFN) - unload(TESTFN) - del sys.path[0] + s = os.stat(fn) + self.assertEqual( + stat.S_IMODE(s.st_mode), + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + finally: + del sys.path[0] + remove_files(TESTFN) + unload(TESTFN) def test_imp_module(self): # Verify that the imp module can correctly load and find .py files @@ -144,10 +153,12 @@ class ImportTests(unittest.TestCase): f.write('"",\n') f.write(']') - # Compile & remove .py file, we only need .pyc (or .pyo). + # Compile & remove .py file, we only need .pyc (or .pyo), but that + # must be relocated to the PEP 3147 bytecode-only location. with open(filename, 'r') as f: py_compile.compile(filename) unlink(filename) + make_legacy_pyc(filename) # Need to be able to load from current dir. sys.path.append('') @@ -247,8 +258,9 @@ class ImportTests(unittest.TestCase): self.assertTrue(mod.__file__.endswith('.py')) os.remove(source) del sys.modules[TESTFN] + make_legacy_pyc(source) mod = __import__(TESTFN) - ext = mod.__file__[-4:] + base, ext = os.path.splitext(mod.__file__) self.assertIn(ext, ('.pyc', '.pyo')) finally: del sys.path[0] @@ -298,7 +310,7 @@ func_filename = func.__code__.co_filename """ dir_name = os.path.abspath(TESTFN) file_name = os.path.join(dir_name, module_name) + os.extsep + "py" - compiled_name = file_name + ("c" if __debug__ else "o") + compiled_name = imp.cache_from_source(file_name) def setUp(self): self.sys_path = sys.path[:] @@ -346,8 +358,9 @@ func_filename = func.__code__.co_filename target = "another_module.py" py_compile.compile(self.file_name, dfile=target) os.remove(self.file_name) + pyc_file = make_legacy_pyc(self.file_name) mod = self.import_module() - self.assertEqual(mod.module_filename, self.compiled_name) + self.assertEqual(mod.module_filename, pyc_file) self.assertEqual(mod.code_filename, target) self.assertEqual(mod.func_filename, target) @@ -476,10 +489,143 @@ class OverridingImportBuiltinTests(unittest.TestCase): self.assertEqual(foo(), os) +class PycacheTests(unittest.TestCase): + # Test the various PEP 3147 related behaviors. + + tag = imp.get_tag() + + def _clean(self): + forget(TESTFN) + rmtree('__pycache__') + unlink(self.source) + + def setUp(self): + self.source = TESTFN + '.py' + self._clean() + with open(self.source, 'w') as fp: + print('# This is a test file written by test_import.py', file=fp) + sys.path.insert(0, os.curdir) + + def tearDown(self): + assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' + del sys.path[0] + self._clean() + + def test_import_pyc_path(self): + self.assertFalse(os.path.exists('__pycache__')) + __import__(TESTFN) + self.assertTrue(os.path.exists('__pycache__')) + self.assertTrue(os.path.exists(os.path.join( + '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag)))) + + @unittest.skipUnless(os.name == 'posix', + "test meaningful only on posix systems") + def test_unwritable_directory(self): + # When the umask causes the new __pycache__ directory to be + # unwritable, the import still succeeds but no .pyc file is written. + with temp_umask(0o222): + __import__(TESTFN) + self.assertTrue(os.path.exists('__pycache__')) + self.assertFalse(os.path.exists(os.path.join( + '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag)))) + + def test_missing_source(self): + # With PEP 3147 cache layout, removing the source but leaving the pyc + # file does not satisfy the import. + __import__(TESTFN) + pyc_file = imp.cache_from_source(self.source) + self.assertTrue(os.path.exists(pyc_file)) + os.remove(self.source) + forget(TESTFN) + self.assertRaises(ImportError, __import__, TESTFN) + + def test_missing_source_legacy(self): + # Like test_missing_source() except that for backward compatibility, + # when the pyc file lives where the py file would have been (and named + # without the tag), it is importable. The __file__ of the imported + # module is the pyc location. + __import__(TESTFN) + # pyc_file gets removed in _clean() via tearDown(). + pyc_file = make_legacy_pyc(self.source) + os.remove(self.source) + unload(TESTFN) + m = __import__(TESTFN) + self.assertEqual(m.__file__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + + def test___cached__(self): + # Modules now also have an __cached__ that points to the pyc file. + m = __import__(TESTFN) + pyc_file = imp.cache_from_source(TESTFN + '.py') + self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file)) + + def test___cached___legacy_pyc(self): + # Like test___cached__() except that for backward compatibility, + # when the pyc file lives where the py file would have been (and named + # without the tag), it is importable. The __cached__ of the imported + # module is the pyc location. + __import__(TESTFN) + # pyc_file gets removed in _clean() via tearDown(). + pyc_file = make_legacy_pyc(self.source) + os.remove(self.source) + unload(TESTFN) + m = __import__(TESTFN) + self.assertEqual(m.__cached__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + + def test_package___cached__(self): + # Like test___cached__ but for packages. + def cleanup(): + shutil.rmtree('pep3147') + os.mkdir('pep3147') + self.addCleanup(cleanup) + # Touch the __init__.py + with open(os.path.join('pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('pep3147', 'foo.py'), 'w'): + pass + unload('pep3147.foo') + unload('pep3147') + m = __import__('pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('pep3147', '__init__.py')) + self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) + foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + self.assertEqual(sys.modules['pep3147.foo'].__cached__, + os.path.join(os.curdir, foo_pyc)) + + def test_package___cached___from_pyc(self): + # Like test___cached__ but ensuring __cached__ when imported from a + # PEP 3147 pyc file. + def cleanup(): + shutil.rmtree('pep3147') + os.mkdir('pep3147') + self.addCleanup(cleanup) + unload('pep3147.foo') + unload('pep3147') + # Touch the __init__.py + with open(os.path.join('pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('pep3147', 'foo.py'), 'w'): + pass + m = __import__('pep3147.foo') + unload('pep3147.foo') + unload('pep3147') + m = __import__('pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('pep3147', '__init__.py')) + self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) + foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + self.assertEqual(sys.modules['pep3147.foo'].__cached__, + os.path.join(os.curdir, foo_pyc)) + + def test_main(verbose=None): - run_unittest(ImportTests, PycRewritingTests, PathsTests, RelativeImportTests, + run_unittest(ImportTests, PycacheTests, + PycRewritingTests, PathsTests, RelativeImportTests, OverridingImportBuiltinTests) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. from test.test_import import test_main diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index 2c195898fea..a342f7a7aee 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -196,14 +196,14 @@ class TestPkg(unittest.TestCase): import t5 self.assertEqual(fixdir(dir(t5)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'foo', 'string', 't5']) self.assertEqual(fixdir(dir(t5.foo)), - ['__doc__', '__file__', '__name__', '__package__', - 'string']) + ['__cached__', '__doc__', '__file__', '__name__', + '__package__', 'string']) self.assertEqual(fixdir(dir(t5.string)), - ['__doc__', '__file__', '__name__','__package__', - 'spam']) + ['__cached__', '__doc__', '__file__', '__name__', + '__package__', 'spam']) def test_6(self): hier = [ @@ -218,13 +218,13 @@ class TestPkg(unittest.TestCase): import t6 self.assertEqual(fixdir(dir(t6)), - ['__all__', '__doc__', '__file__', + ['__all__', '__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) s = """ import t6 from t6 import * self.assertEqual(fixdir(dir(t6)), - ['__all__', '__doc__', '__file__', + ['__all__', '__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'eggs', 'ham', 'spam']) self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6']) @@ -252,18 +252,18 @@ class TestPkg(unittest.TestCase): t7, sub, subsub = None, None, None import t7 as tas self.assertEqual(fixdir(dir(tas)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) self.assertFalse(t7) from t7 import sub as subpar self.assertEqual(fixdir(dir(subpar)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) self.assertFalse(t7) self.assertFalse(sub) from t7.sub import subsub as subsubsub self.assertEqual(fixdir(dir(subsubsub)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'spam']) self.assertFalse(t7) self.assertFalse(sub) diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py index a9a475c3c61..eab66fb1580 100644 --- a/Lib/test/test_pkgimport.py +++ b/Lib/test/test_pkgimport.py @@ -1,5 +1,12 @@ -import os, sys, string, random, tempfile, unittest +import os +import sys +import shutil +import string +import random +import tempfile +import unittest +from imp import cache_from_source from test.support import run_unittest class TestImport(unittest.TestCase): @@ -26,22 +33,17 @@ class TestImport(unittest.TestCase): self.module_path = os.path.join(self.package_dir, 'foo.py') def tearDown(self): - for file in os.listdir(self.package_dir): - os.remove(os.path.join(self.package_dir, file)) - os.rmdir(self.package_dir) - os.rmdir(self.test_dir) + shutil.rmtree(self.test_dir) self.assertNotEqual(sys.path.count(self.test_dir), 0) sys.path.remove(self.test_dir) self.remove_modules() def rewrite_file(self, contents): - for extension in "co": - compiled_path = self.module_path + extension - if os.path.exists(compiled_path): - os.remove(compiled_path) - f = open(self.module_path, 'w') - f.write(contents) - f.close() + compiled_path = cache_from_source(self.module_path) + if os.path.exists(compiled_path): + os.remove(compiled_path) + with open(self.module_path, 'w') as f: + f.write(contents) def test_package_import__semantics(self): diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index d0b81e35d88..603755ad0ee 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -19,8 +19,7 @@ from test import pydoc_mod if hasattr(pydoc_mod, "__loader__"): del pydoc_mod.__loader__ -expected_text_pattern = \ -""" +expected_text_pattern = """ NAME test.pydoc_mod - This is a test module for test_pydoc @@ -87,8 +86,7 @@ CREDITS Nobody """.strip() -expected_html_pattern = \ -""" +expected_html_pattern = """
 
@@ -186,7 +184,7 @@ war \x20\x20\x20\x20
        Nobody
-""".strip() +""".strip() # ' <- emacs turd # output pattern for missing module @@ -287,7 +285,8 @@ class PyDocDocTest(unittest.TestCase): ('i_am_not_here', 'i_am_not_here'), ('test.i_am_not_here_either', 'i_am_not_here_either'), ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'), - ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)), + ('i_am_not_here.{}'.format(modname), + 'i_am_not_here.{}'.format(modname)), ('test.{}'.format(modname), modname), ) @@ -304,9 +303,8 @@ class PyDocDocTest(unittest.TestCase): fullmodname = os.path.join(TESTFN, modname) sourcefn = fullmodname + os.extsep + "py" for importstring, expectedinmsg in testpairs: - f = open(sourcefn, 'w') - f.write("import {}\n".format(importstring)) - f.close() + with open(sourcefn, 'w') as f: + f.write("import {}\n".format(importstring)) try: result = run_pydoc(modname).decode("ascii") finally: diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 995c8917d5f..068eca94a02 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -5,9 +5,10 @@ import os.path import sys import re import tempfile -from test.support import verbose, run_unittest, forget -from test.script_helper import (temp_dir, make_script, compile_script, - make_pkg, make_zip_script, make_zip_pkg) +import py_compile +from test.support import forget, make_legacy_pyc, run_unittest, verbose +from test.script_helper import ( + make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir) from runpy import _run_code, _run_module_code, run_module, run_path @@ -45,6 +46,7 @@ class RunModuleCodeTest(unittest.TestCase): self.assertEqual(d["result"], self.expected_result) self.assertIs(d["__name__"], None) self.assertIs(d["__file__"], None) + self.assertIs(d["__cached__"], None) self.assertIs(d["__loader__"], None) self.assertIs(d["__package__"], None) self.assertIs(d["run_argv0"], saved_argv0) @@ -73,6 +75,7 @@ class RunModuleCodeTest(unittest.TestCase): self.assertTrue(d2["run_name_in_sys_modules"]) self.assertTrue(d2["module_in_sys_modules"]) self.assertIs(d2["__file__"], file) + self.assertIs(d2["__cached__"], None) self.assertIs(d2["run_argv0"], file) self.assertIs(d2["__loader__"], loader) self.assertIs(d2["__package__"], package) @@ -170,6 +173,7 @@ class RunModuleTest(unittest.TestCase): del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", mod_name) d2 = run_module(mod_name) # Read from bytecode self.assertIn("x", d2) @@ -192,6 +196,7 @@ class RunModuleTest(unittest.TestCase): del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", pkg_name) d2 = run_module(pkg_name) # Read from bytecode self.assertIn("x", d2) @@ -246,6 +251,7 @@ from ..uncle.cousin import nephew del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", mod_name) d2 = run_module(mod_name, run_name=run_name) # Read from bytecode self.assertIn("__package__", d2) @@ -313,6 +319,7 @@ argv0 = sys.argv[0] result = run_path(script_name) self.assertEqual(result["__name__"], expected_name) self.assertEqual(result["__file__"], expected_file) + self.assertEqual(result["__cached__"], None) self.assertIn("argv0", result) self.assertEqual(result["argv0"], expected_argv0) self.assertEqual(result["__package__"], expected_package) @@ -332,7 +339,7 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = 'script' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) self._check_script(compiled_name, "", compiled_name, compiled_name, None) @@ -348,9 +355,10 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(script_dir, "", compiled_name, + legacy_pyc = make_legacy_pyc(script_name) + self._check_script(script_dir, "", legacy_pyc, script_dir, '') def test_directory_error(self): @@ -371,8 +379,9 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) - zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) + compiled_name = py_compile.compile(script_name, doraise=True) + zip_name, fname = make_zip_script(script_dir, 'test_zip', + compiled_name) self._check_script(zip_name, "", fname, zip_name, '') def test_zipfile_error(self): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 931a166f1c0..1a50f191334 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -258,19 +258,38 @@ class ImportSideEffectTests(unittest.TestCase): """Restore sys.path""" sys.path[:] = self.sys_path - def test_abs__file__(self): - # Make sure all imported modules have their __file__ attribute - # as an absolute path. - # Handled by abs__file__() - site.abs__file__() - for module in (sys, os, builtins): - try: - self.assertTrue(os.path.isabs(module.__file__), repr(module)) - except AttributeError: - continue - # We could try everything in sys.modules; however, when regrtest.py - # runs something like test_frozen before test_site, then we will - # be testing things loaded *after* test_site did path normalization + def test_abs_paths(self): + # Make sure all imported modules have their __file__ and __cached__ + # attributes as absolute paths. Arranging to put the Lib directory on + # PYTHONPATH would cause the os module to have a relative path for + # __file__ if abs_paths() does not get run. sys and builtins (the + # only other modules imported before site.py runs) do not have + # __file__ or __cached__ because they are built-in. + parent = os.path.relpath(os.path.dirname(os.__file__)) + env = os.environ.copy() + env['PYTHONPATH'] = parent + command = 'import os; print(os.__file__, os.__cached__)' + # First, prove that with -S (no 'import site'), the paths are + # relative. + proc = subprocess.Popen([sys.executable, '-S', '-c', command], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + self.assertEqual(proc.returncode, 0) + os__file__, os__cached__ = stdout.split() + self.assertFalse(os.path.isabs(os__file__)) + self.assertFalse(os.path.isabs(os__cached__)) + # Now, with 'import site', it works. + proc = subprocess.Popen([sys.executable, '-c', command], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + self.assertEqual(proc.returncode, 0) + os__file__, os__cached__ = stdout.split() + self.assertTrue(os.path.isabs(os__file__)) + self.assertTrue(os.path.isabs(os__cached__)) def test_no_duplicate_paths(self): # No duplicate paths should exist in sys.path diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 8e2cf553e7e..e9a90e55fbc 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -6,6 +6,7 @@ except ImportError: import io import os +import imp import time import shutil import struct @@ -587,7 +588,13 @@ class PyZipFileTests(unittest.TestCase): with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp: fn = __file__ if fn.endswith('.pyc') or fn.endswith('.pyo'): - fn = fn[:-1] + path_split = fn.split(os.sep) + if os.altsep is not None: + path_split.extend(fn.split(os.altsep)) + if '__pycache__' in path_split: + fn = imp.source_from_cache(fn) + else: + fn = fn[:-1] zipfp.writepy(fn) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index c89aef5799f..ba4e34a9607 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -48,17 +48,14 @@ NOW = time.time() test_pyc = make_pyc(test_co, NOW) -if __debug__: - pyc_ext = ".pyc" -else: - pyc_ext = ".pyo" - - TESTMOD = "ziptestmodule" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" TEMP_ZIP = os.path.abspath("junk95142.zip") +pyc_file = imp.cache_from_source(TESTMOD + '.py') +pyc_ext = ('.pyc' if __debug__ else '.pyo') + class UncompressedZipImportTestCase(ImportHooksBaseTestCase): @@ -83,14 +80,11 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile - f = open(TEMP_ZIP, "rb") - data = f.read() - f.close() - - f = open(TEMP_ZIP, "wb") - f.write(stuff) - f.write(data) - f.close() + with open(TEMP_ZIP, "rb") as f: + data = f.read() + with open(TEMP_ZIP, "wb") as f: + f.write(stuff) + f.write(data) sys.path.insert(0, TEMP_ZIP) @@ -180,8 +174,9 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): def testBadMTime(self): badtime_pyc = bytearray(test_pyc) - badtime_pyc[7] ^= 0x02 # flip the second bit -- not the first as that one - # isn't stored in the .py's mtime in the zip archive. + # flip the second bit -- not the first as that one isn't stored in the + # .py's mtime in the zip archive. + badtime_pyc[7] ^= 0x02 files = {TESTMOD + ".py": (NOW, test_src), TESTMOD + pyc_ext: (NOW, badtime_pyc)} self.doTest(".py", files, TESTMOD) @@ -232,7 +227,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEquals(zi.get_source(TESTPACK), None) self.assertEquals(zi.get_source(mod_path), None) self.assertEquals(zi.get_filename(mod_path), mod.__file__) - # To pass in the module name instead of the path, we must use the right importer + # To pass in the module name instead of the path, we must use the + # right importer loader = mod.__loader__ self.assertEquals(loader.get_source(mod_name), None) self.assertEquals(loader.get_filename(mod_name), mod.__file__) @@ -266,8 +262,10 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): mod = zi.load_module(TESTPACK2) self.assertEquals(zi.get_filename(TESTPACK2), mod.__file__) - self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False) - self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) + self.assertEquals( + zi.is_package(TESTPACK2 + os.sep + '__init__'), False) + self.assertEquals( + zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) @@ -276,7 +274,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEquals(zi.get_source(TESTPACK2), None) self.assertEquals(zi.get_source(mod_path), None) self.assertEquals(zi.get_filename(mod_path), mod.__file__) - # To pass in the module name instead of the path, we must use the right importer + # To pass in the module name instead of the path, we must use the + # right importer loader = mod.__loader__ self.assertEquals(loader.get_source(mod_name), None) self.assertEquals(loader.get_filename(mod_name), mod.__file__) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 2982ec31c93..f81cc8b2eee 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -3,10 +3,17 @@ Read and write ZIP files. XXX references to utf-8 need further investigation. """ -import struct, os, time, sys, shutil -import binascii, io, stat import io +import os import re +import imp +import sys +import time +import stat +import shutil +import struct +import binascii + try: import zlib # We may need its compression method @@ -1303,22 +1310,42 @@ class PyZipFile(ZipFile): file_py = pathname + ".py" file_pyc = pathname + ".pyc" file_pyo = pathname + ".pyo" - if os.path.isfile(file_pyo) and \ - os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime: - fname = file_pyo # Use .pyo file - elif not os.path.isfile(file_pyc) or \ - os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime: + pycache_pyc = imp.cache_from_source(file_py, True) + pycache_pyo = imp.cache_from_source(file_py, False) + if (os.path.isfile(file_pyo) and + os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyo file. + arcname = fname = file_pyo + elif (os.path.isfile(file_pyc) and + os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyc file. + arcname = fname = file_pyc + elif (os.path.isfile(pycache_pyc) and + os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_pyc + arcname = file_pyc + elif (os.path.isfile(pycache_pyo) and + os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyo file, but write it to the legacy pyo + # file name in the archive. + fname = pycache_pyo + arcname = file_pyo + else: + # Compile py into PEP 3147 pyc file. import py_compile if self.debug: print("Compiling", file_py) try: - py_compile.compile(file_py, file_pyc, None, True) - except py_compile.PyCompileError as err: + py_compile.compile(file_py, doraise=True) + except py_compile.PyCompileError as error: print(err.msg) - fname = file_pyc - else: - fname = file_pyc - archivename = os.path.split(fname)[1] + fname = file_py + else: + fname = (pycache_pyc if __debug__ else pycache_pyo) + arcname = (file_pyc if __debug__ else file_pyo) + archivename = os.path.split(arcname)[1] if basename: archivename = "%s/%s" % (basename, archivename) return (fname, archivename) diff --git a/Makefile.pre.in b/Makefile.pre.in index e3fdf410ec1..a5e8787ccbd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1161,6 +1161,7 @@ TAGS:: # files, which clobber removes as well pycremoval: find $(srcdir) -name '*.py[co]' -exec rm -f {} ';' + find $(srcdir) -name '__pycache__' | xargs rmdir rmtestturds: -rm -f *BAD *GOOD *SKIPPED diff --git a/Python/import.c b/Python/import.c index b0463628188..1cddcb06584 100644 --- a/Python/import.c +++ b/Python/import.c @@ -43,6 +43,15 @@ typedef unsigned short mode_t; The current working scheme is to increment the previous value by 10. + Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic + number also includes a new "magic tag", i.e. a human readable string used + to represent the magic number in __pycache__ directories. When you change + the magic number, you must also set a new unique magic tag. Generally this + can be named after the Python major version of the magic number bump, but + it can really be anything, as long as it's different than anything else + that's come before. The tags are included in the following table, starting + with Python 3.2a0. + Known values: Python 1.5: 20121 Python 1.5.1: 20121 @@ -91,11 +100,18 @@ typedef unsigned short mode_t; Python 3.1a0: 3151 (optimize conditional branches: introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) Python 3.2a0: 3160 (add SETUP_WITH) + tag: cpython-32 */ +/* If you change MAGIC, you must change TAG and you must insert the old value + into _PyMagicNumberTags below. +*/ #define MAGIC (3160 | ((long)'\r'<<16) | ((long)'\n'<<24)) -/* Magic word as global */ +#define TAG "cpython-32" +#define CACHEDIR "__pycache__" +/* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; +static const char *pyc_tag = TAG; /* See _PyImport_FixupExtension() below */ static PyObject *extensions = NULL; @@ -517,7 +533,7 @@ PyImport_Cleanup(void) } -/* Helper for pythonrun.c -- return magic number */ +/* Helper for pythonrun.c -- return magic number and tag. */ long PyImport_GetMagicNumber(void) @@ -526,6 +542,12 @@ PyImport_GetMagicNumber(void) } +const char * +PyImport_GetMagicTag(void) +{ + return pyc_tag; +} + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by module name @@ -671,7 +693,10 @@ remove_module(const char *name) "sys.modules failed"); } -static PyObject * get_sourcefile(const char *file); +static PyObject * get_sourcefile(char *file); +static char *make_source_pathname(char *pathname, char *buf); +static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, + int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -679,15 +704,27 @@ static PyObject * get_sourcefile(const char *file); * in sys.modules. The caller may wish to restore the original * module object (if any) in this case; PyImport_ReloadModule is an * example. + * + * Note that PyImport_ExecCodeModuleWithPathnames() is the preferred, richer + * interface. The other two exist primarily for backward compatibility. */ PyObject * PyImport_ExecCodeModule(char *name, PyObject *co) { - return PyImport_ExecCodeModuleEx(name, co, (char *)NULL); + return PyImport_ExecCodeModuleWithPathnames( + name, co, (char *)NULL, (char *)NULL); } PyObject * PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) +{ + return PyImport_ExecCodeModuleWithPathnames( + name, co, pathname, (char *)NULL); +} + +PyObject * +PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, + char *cpathname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; @@ -718,6 +755,20 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) PyErr_Clear(); /* Not important enough to report */ Py_DECREF(v); + /* Remember the pyc path name as the __cached__ attribute. */ + if (cpathname == NULL) { + v = Py_None; + Py_INCREF(v); + } + else if ((v = PyUnicode_FromString(cpathname)) == NULL) { + PyErr_Clear(); /* Not important enough to report */ + v = Py_None; + Py_INCREF(v); + } + if (PyDict_SetItemString(d, "__cached__", v) != 0) + PyErr_Clear(); /* Not important enough to report */ + Py_DECREF(v); + v = PyEval_EvalCode((PyCodeObject *)co, d, d); if (v == NULL) goto error; @@ -740,32 +791,189 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) } +/* Like strrchr(string, '/') but searches for the rightmost of either SEP + or ALTSEP, if the latter is defined. +*/ +static char * +rightmost_sep(char *s) +{ + char *found, c; + for (found = NULL; (c = *s); s++) { + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = s; + } + } + return found; +} + + /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. Doesn't set an exception. */ static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen) +make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) { + /* foo.py -> __pycache__/foo..pyc */ size_t len = strlen(pathname); - if (len+2 > buflen) + size_t i, save; + char *pos; + int sep = SEP; + + /* Sanity check that the buffer has roughly enough space to hold what + will eventually be the full path to the compiled file. The 5 extra + bytes include the slash afer __pycache__, the two extra dots, the + extra trailing character ('c' or 'o') and null. This isn't exact + because the contents of the buffer can affect how many actual + characters of the string get into the buffer. We'll do a final + sanity check before writing the extension to ensure we do not + overflow the buffer. + */ + if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) return NULL; -#ifdef MS_WINDOWS - /* Treat .pyw as if it were .py. The case of ".pyw" must match - that used in _PyImport_StandardFiletab. */ - if (len >= 4 && strcmp(&pathname[len-4], ".pyw") == 0) - --len; /* pretend 'w' isn't there */ -#endif - memcpy(buf, pathname, len); - buf[len] = Py_OptimizeFlag ? 'o' : 'c'; - buf[len+1] = '\0'; + /* Find the last path separator and copy everything from the start of + the source string up to and including the separator. + */ + if ((pos = rightmost_sep(pathname)) == NULL) { + i = 0; + } + else { + sep = *pos; + i = pos - pathname + 1; + strncpy(buf, pathname, i); + } + save = i; + buf[i++] = '\0'; + /* Add __pycache__/ */ + strcat(buf, CACHEDIR); + i += strlen(CACHEDIR) - 1; + buf[i++] = sep; + buf[i++] = '\0'; + /* Add the base filename, but remove the .py or .pyw extension, since + the tag name must go before the extension. + */ + strcat(buf, pathname + save); + if ((pos = strrchr(buf, '.')) != NULL) + *++pos = '\0'; + strcat(buf, pyc_tag); + /* The length test above assumes that we're only adding one character + to the end of what would normally be the extension. What if there + is no extension, or the string ends in '.' or '.p', and otherwise + fills the buffer? By appending 4 more characters onto the string + here, we could overrun the buffer. + + As a simple example, let's say buflen=32 and the input string is + 'xxx.py'. strlen() would be 6 and the test above would yield: + + (6 + 11 + 10 + 5 == 32) > 32 + + which is false and so the name mangling would continue. This would + be fine because we'd end up with this string in buf: + + __pycache__/xxx.cpython-32.pyc\0 + + strlen(of that) == 30 + the nul fits inside a 32 character buffer. + We can even handle an input string of say 'xxxxx' above because + that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name + mangling that yields: + + __pycache__/xxxxxcpython-32.pyc\0 + + which is 32 characters including the nul, and thus fits in the + buffer. However, an input string of 'xxxxxx' would yield a result + string of: + + __pycache__/xxxxxxcpython-32.pyc\0 + + which is 33 characters long (including the nul), thus overflowing + the buffer, even though the first test would fail, i.e.: the input + string is also 6 characters long, so 32 > 32 is false. + + The reason the first test fails but we still overflow the buffer is + that the test above only expects to add one extra character to be + added to the extension, and here we're adding three (pyc). We + don't add the first dot, so that reclaims one of expected + positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed + dot - 1 expected extra == 1 overflowed). + + The best we can do is ensure that we still have enough room in the + target buffer before we write the extension. Because it's always + only the extension that can cause the overflow, and never the other + path bytes we've written, it's sufficient to just do one more test + here. Still, the assertion that follows can't hurt. + */ +#if 0 + printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); +#endif + if (strlen(buf) + 5 > buflen) + return NULL; + strcat(buf, debug ? ".pyc" : ".pyo"); + assert(strlen(buf) < buflen); return buf; } +/* Given a pathname to a Python byte compiled file, return the path to the + source file, if the path matches the PEP 3147 format. This does not check + for any file existence, however, if the pyc file name does not match PEP + 3147 style, NULL is returned. buf must be at least as big as pathname; + the resulting path will always be shorter. */ + +static char * +make_source_pathname(char *pathname, char *buf) +{ + /* __pycache__/foo..pyc -> foo.py */ + size_t i, j; + char *left, *right, *dot0, *dot1, sep; + + /* Look back two slashes from the end. In between these two slashes + must be the string __pycache__ or this is not a PEP 3147 style + path. It's possible for there to be only one slash. + */ + if ((right = rightmost_sep(pathname)) == NULL) + return NULL; + sep = *right; + *right = '\0'; + left = rightmost_sep(pathname); + *right = sep; + if (left == NULL) + left = pathname; + else + left++; + if (right-left != strlen(CACHEDIR) || + strncmp(left, CACHEDIR, right-left) != 0) + return NULL; + + /* Now verify that the path component to the right of the last slash + has two dots in it. + */ + if ((dot0 = strchr(right + 1, '.')) == NULL) + return NULL; + if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + return NULL; + /* Too many dots? */ + if (strchr(dot1 + 1, '.') != NULL) + return NULL; + + /* This is a PEP 3147 path. Start by copying everything from the + start of pathname up to and including the leftmost slash. Then + copy the file's basename, removing the magic tag and adding a .py + suffix. + */ + strncpy(buf, pathname, (i=left-pathname)); + strncpy(buf+i, right+1, (j=dot0-right)); + strcpy(buf+i+j, "py"); + return buf; +} + /* Given a pathname for a Python source file, its time of last modification, and a pathname for a compiled file, check whether the compiled file represents the same version of the source. If so, @@ -846,7 +1054,8 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) if (Py_VerboseFlag) PySys_WriteStderr("import %s # precompiled from %s\n", name, cpathname); - m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, cpathname); + m = PyImport_ExecCodeModuleWithPathnames( + name, (PyObject *)co, cpathname, cpathname); Py_DECREF(co); return m; @@ -919,12 +1128,41 @@ static void write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) { FILE *fp; + char *dirpath; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + mode_t dirmode = srcstat->st_mode | S_IEXEC; /* XXX Is this correct + for Windows? + 2010-04-07 BAW */ #else mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; + mode_t dirmode = (srcstat->st_mode | + S_IXUSR | S_IXGRP | S_IXOTH | + S_IWUSR | S_IWGRP | S_IWOTH); #endif + int saved; + + /* Ensure that the __pycache__ directory exists. */ + dirpath = rightmost_sep(cpathname); + if (dirpath == NULL) { + if (Py_VerboseFlag) + PySys_WriteStderr( + "# no %s path found %s\n", + CACHEDIR, cpathname); + return; + } + saved = *dirpath; + *dirpath = '\0'; + /* XXX call os.mkdir() or maybe CreateDirectoryA() on Windows? */ + if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { + *dirpath = saved; + if (Py_VerboseFlag) + PySys_WriteStderr( + "# cannot create cache dir %s\n", cpathname); + return; + } + *dirpath = saved; fp = open_exclusive(cpathname, mode); if (fp == NULL) { @@ -1032,8 +1270,8 @@ load_source_module(char *name, char *pathname, FILE *fp) return NULL; } #endif - cpathname = make_compiled_pathname(pathname, buf, - (size_t)MAXPATHLEN + 1); + cpathname = make_compiled_pathname( + pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); if (cpathname != NULL && (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { co = read_compiled_module(cpathname, fpc); @@ -1060,7 +1298,8 @@ load_source_module(char *name, char *pathname, FILE *fp) write_compiled_module(co, cpathname, &st); } } - m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname); + m = PyImport_ExecCodeModuleWithPathnames( + name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); return m; @@ -1070,7 +1309,7 @@ load_source_module(char *name, char *pathname, FILE *fp) * Returns the path to the py file if available, else the given path */ static PyObject * -get_sourcefile(const char *file) +get_sourcefile(char *file) { char py[MAXPATHLEN + 1]; Py_ssize_t len; @@ -1087,8 +1326,15 @@ get_sourcefile(const char *file) return PyUnicode_DecodeFSDefault(file); } - strncpy(py, file, len-1); - py[len-1] = '\0'; + /* Start by trying to turn PEP 3147 path into source path. If that + * fails, just chop off the trailing character, i.e. legacy pyc path + * to py. + */ + if (make_source_pathname(file, py) == NULL) { + strncpy(py, file, len-1); + py[len-1] = '\0'; + } + if (stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { u = PyUnicode_DecodeFSDefault(py); @@ -2813,16 +3059,28 @@ PyImport_Import(PyObject *module_name) */ static PyObject * -imp_get_magic(PyObject *self, PyObject *noargs) +imp_make_magic(long magic) { char buf[4]; - buf[0] = (char) ((pyc_magic >> 0) & 0xff); - buf[1] = (char) ((pyc_magic >> 8) & 0xff); - buf[2] = (char) ((pyc_magic >> 16) & 0xff); - buf[3] = (char) ((pyc_magic >> 24) & 0xff); + buf[0] = (char) ((magic >> 0) & 0xff); + buf[1] = (char) ((magic >> 8) & 0xff); + buf[2] = (char) ((magic >> 16) & 0xff); + buf[3] = (char) ((magic >> 24) & 0xff); return PyBytes_FromStringAndSize(buf, 4); +}; + +static PyObject * +imp_get_magic(PyObject *self, PyObject *noargs) +{ + return imp_make_magic(pyc_magic); +} + +static PyObject * +imp_get_tag(PyObject *self, PyObject *noargs) +{ + return PyUnicode_FromString(pyc_tag); } static PyObject * @@ -3190,6 +3448,75 @@ PyDoc_STRVAR(doc_reload, \n\ Reload the module. The module must have been successfully imported before."); +static PyObject * +imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) +{ + static char *kwlist[] = {"path", "debug_override", NULL}; + + char buf[MAXPATHLEN+1]; + char *pathname, *cpathname; + PyObject *debug_override = Py_None; + int debug = !Py_OptimizeFlag; + + if (!PyArg_ParseTupleAndKeywords( + args, kws, "es|O", kwlist, + Py_FileSystemDefaultEncoding, &pathname, &debug_override)) + return NULL; + + if (debug_override != Py_None) + if ((debug = PyObject_IsTrue(debug_override)) < 0) + return NULL; + + cpathname = make_compiled_pathname(pathname, buf, MAXPATHLEN+1, debug); + PyMem_Free(pathname); + + if (cpathname == NULL) { + PyErr_Format(PyExc_SystemError, "path buffer too short"); + return NULL; + } + return PyUnicode_FromString(buf); +} + +PyDoc_STRVAR(doc_cache_from_source, +"Given the path to a .py file, return the path to its .pyc/.pyo file.\n\ +\n\ +The .py file does not need to exist; this simply returns the path to the\n\ +.pyc/.pyo file calculated as if the .py file were imported. The extension\n\ +will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\ +\n\ +If debug_override is not None, then it must be a boolean and is taken as\n\ +the value of __debug__ instead."); + +static PyObject * +imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) +{ + static char *kwlist[] = {"path", NULL}; + + char *pathname; + char buf[MAXPATHLEN+1]; + + if (!PyArg_ParseTupleAndKeywords( + args, kws, "es", kwlist, + Py_FileSystemDefaultEncoding, &pathname)) + return NULL; + + if (make_source_pathname(pathname, buf) == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + pathname); + PyMem_Free(pathname); + return NULL; + } + PyMem_Free(pathname); + return PyUnicode_FromString(buf); +} + +PyDoc_STRVAR(doc_source_from_cache, +"Given the path to a .pyc./.pyo file, return the path to its .py file.\n\ +\n\ +The .pyc/.pyo file does not need to exist; this simply returns the path to\n\ +the .py file calculated to correspond to the .pyc/.pyo file. If path\n\ +does not conform to PEP 3147 format, ValueError will be raised."); + /* Doc strings */ PyDoc_STRVAR(doc_imp, @@ -3212,6 +3539,10 @@ PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); +PyDoc_STRVAR(doc_get_tag, +"get_tag() -> string\n\ +Return the magic tag for .pyc or .pyo files."); + PyDoc_STRVAR(doc_get_suffixes, "get_suffixes() -> [(suffix, mode, type), ...]\n\ Return a list of (suffix, mode, type) tuples describing the files\n\ @@ -3242,6 +3573,7 @@ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { {"find_module", imp_find_module, METH_VARARGS, doc_find_module}, {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, + {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, {"new_module", imp_new_module, METH_VARARGS, doc_new_module}, @@ -3249,6 +3581,10 @@ static PyMethodDef imp_methods[] = { {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, {"reload", imp_reload, METH_O, doc_reload}, + {"cache_from_source", (PyCFunction)imp_cache_from_source, + METH_VARARGS | METH_KEYWORDS, doc_cache_from_source}, + {"source_from_cache", (PyCFunction)imp_source_from_cache, + METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, /* The rest are obsolete */ {"get_frozen_object", imp_get_frozen_object, METH_VARARGS}, {"is_frozen_package", imp_is_frozen_package, METH_VARARGS}, @@ -3436,7 +3772,6 @@ PyInit_imp(void) failure: Py_XDECREF(m); return NULL; - } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2bdef981ad5..cc617be4d11 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1155,6 +1155,8 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, Py_DECREF(f); return -1; } + if (PyDict_SetItemString(d, "__cached__", Py_None) < 0) + return -1; set_file_name = 1; Py_DECREF(f); }