From 49868cb686954e57a6ab6bfea2cefeefb86305b1 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 15 Nov 2009 07:30:34 +0000 Subject: [PATCH] Issue #6816: expose the zipfile and directory execution mechanism to Python code via the runpy module. Also consolidated some script execution functionality in the test harness into a helper module and removed some implementation details from the runpy module documentation. --- Doc/library/runpy.rst | 141 +++++++++++++++------- Lib/runpy.py | 187 +++++++++++++++++++++++------ Lib/test/script_helper.py | 119 ++++++++++++++++++ Lib/test/test_cmd_line.py | 30 +---- Lib/test/test_cmd_line_script.py | 121 ++++--------------- Lib/test/test_runpy.py | 164 +++++++++++++++++++++---- Lib/test/test_zipimport_support.py | 36 +++--- Misc/NEWS | 6 + 8 files changed, 563 insertions(+), 241 deletions(-) create mode 100644 Lib/test/script_helper.py diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 202f4e1e358..33b00dc931d 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -9,71 +9,123 @@ .. versionadded:: 2.5 The :mod:`runpy` module is used to locate and run Python modules without -importing them first. Its main use is to implement the :option:`-m` command line -switch that allows scripts to be located using the Python module namespace -rather than the filesystem. +importing them first. Its main use is to implement the :option:`-m` command +line switch that allows scripts to be located using the Python module +namespace rather than the filesystem. -When executed as a script, the module effectively operates as follows:: - - del sys.argv[0] # Remove the runpy module from the arguments - run_module(sys.argv[0], run_name="__main__", alter_sys=True) - -The :mod:`runpy` module provides a single function: +The :mod:`runpy` module provides two functions: -.. function:: run_module(mod_name[, init_globals] [, run_name][, alter_sys]) +.. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) - Execute the code of the specified module and return the resulting module globals - dictionary. The module's code is first located using the standard import - mechanism (refer to PEP 302 for details) and then executed in a fresh module - namespace. + Execute the code of the specified module and return the resulting module + globals dictionary. The module's code is first located using the standard + import mechanism (refer to PEP 302 for details) and then executed in a + fresh module namespace. - If the supplied module name refers to a package rather than a normal module, - then that package is imported and the ``__main__`` submodule within that - package is then executed and the resulting module globals dictionary returned. + If the supplied module name refers to a package rather than a normal + module, then that package is imported and the ``__main__`` submodule within + that package is then executed and the resulting module globals dictionary + returned. - The optional dictionary argument *init_globals* may be used to pre-populate the - globals dictionary before the code is executed. The supplied dictionary will not - be modified. If any of the special global variables below are defined in the - supplied dictionary, those definitions are overridden by the ``run_module`` - function. + The optional dictionary argument *init_globals* may be used to pre-populate + the module's globals dictionary before the code is executed. The supplied + dictionary will not be modified. If any of the special global variables + below are defined in the supplied dictionary, those definitions are + overridden by :func:`run_module`. - The special global variables ``__name__``, ``__file__``, ``__loader__``, - ``__builtins__`` and ``__package__`` are set in the globals dictionary before - the module code is executed. + The special global variables ``__name__``, ``__file__``, ``__loader__`` + and ``__package__`` are set in the globals dictionary before the module + code is executed (Note that this is a minimal set of variables - other + variables may be set implicitly as an interpreter implementation detail). - ``__name__`` is set to *run_name* if this optional argument is supplied, to - ``mod_name + '.__main__'`` if the named module is a package and to the - *mod_name* argument otherwise. + ``__name__`` is set to *run_name* if this optional argument is not + :const:`None`, to ``mod_name + '.__main__'`` if the named module is a + package and to the *mod_name* argument otherwise. - ``__loader__`` is set to the PEP 302 module loader used to retrieve the code for - the module (This loader may be a wrapper around the standard import mechanism). + ``__file__`` is set to the name provided by the module loader. If the + loader does not make filename information available, this variable is set + to `:const:`None`. - ``__file__`` is set to the name provided by the module loader. If the loader - does not make filename information available, this variable is set to ``None``. + ``__loader__`` is set to the PEP 302 module loader used to retrieve the + code for the module (This loader may be a wrapper around the standard + import mechanism). - ``__builtins__`` is automatically initialised with a reference to the top level - namespace of the :mod:`__builtin__` module. + ``__package__`` is set to *mod_name* if the named module is a package and + to ``mod_name.rpartition('.')[0]`` otherwise. - ``__package__`` is set to *mod_name* if the named module is a package and to - ``mod_name.rpartition('.')[0]`` otherwise. - - If the argument *alter_sys* is supplied and evaluates to ``True``, then - ``sys.argv[0]`` is updated with the value of ``__file__`` and + If the argument *alter_sys* is supplied and evaluates to :const:`True`, + then ``sys.argv[0]`` is updated with the value of ``__file__`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. Both ``sys.argv[0]`` and ``sys.modules[__name__]`` are restored to their original values before the function returns. - Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may - see the partially initialised module, as well as the altered list of arguments. - It is recommended that the :mod:`sys` module be left alone when invoking this - function from threaded code. + Note that this manipulation of :mod:`sys` is not thread-safe. Other threads + may see the partially initialised module, as well as the altered list of + arguments. It is recommended that the :mod:`sys` module be left alone when + invoking this function from threaded code. .. versionchanged:: 2.7 - Added ability to execute packages by looking for a ``__main__`` submodule + Added ability to execute packages by looking for a ``__main__`` + submodule +.. function:: run_path(file_path, init_globals=None, run_name=None) + + Execute the code at the named filesystem location and return the resulting + module globals dictionary. As with a script name supplied to the CPython + command line, the supplied path may refer to a Python source file, a + compiled bytecode file or a valid sys.path entry containing a ``__main__`` + module (e.g. a zipfile containing a top-level ``__main__.py`` file). + + For a simple script, the specified code is simply executed in a fresh + module namespace. For a valid sys.path entry (typically a zipfile or + directory), the entry is first added to the beginning of ``sys.path``. The + function then looks for and executes a :mod:`__main__` module using the + updated path. Note that there is no special protection against invoking + an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + there is no such module at the specified location. + + The optional dictionary argument *init_globals* may be used to pre-populate + the module's globals dictionary before the code is executed. The supplied + dictionary will not be modified. If any of the special global variables + below are defined in the supplied dictionary, those definitions are + overridden by :func:`run_path`. + + The special global variables ``__name__``, ``__file__``, ``__loader__`` + and ``__package__`` are set in the globals dictionary before the module + code is executed (Note that this is a minimal set of variables - other + variables may be set implicitly as an interpreter implementation detail). + + ``__name__`` is set to *run_name* if this optional argument is not + :const:`None` and to ``''`` otherwise. + + ``__file__`` is set to the name provided by the module loader. If the + loader does not make filename information available, this variable is set + to :const:`None`. For a simple script, this will be set to ``file_path``. + + ``__loader__`` is set to the PEP 302 module loader used to retrieve the + code for the module (This loader may be a wrapper around the standard + import mechanism). For a simple script, this will be set to :const:`None`. + + ``__package__`` is set to ``__name__.rpartition('.')[0]``. + + A number of alterations are also made to the :mod:`sys` module. Firstly, + ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + with the value of ``file_path`` and ``sys.modules[__name__]`` is updated + with a temporary module object for the module being executed. All + modifications to items in :mod:`sys` are reverted before the function + returns. + + Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` + are not optional in this function as these adjustments are essential to + allowing the execution of sys.path entries. As the thread safety + limitations still apply, use of this function in threaded code should be + either serialised with the import lock or delegated to a separate process. + + .. versionadded:: 2.7 + .. seealso:: :pep:`338` - Executing modules as scripts @@ -82,3 +134,4 @@ The :mod:`runpy` module provides a single function: :pep:`366` - Main module explicit relative imports PEP written and implemented by Nick Coghlan. + :ref:`using-on-general` - CPython command line details diff --git a/Lib/runpy.py b/Lib/runpy.py index d3548025515..50d19f5539c 100755 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -11,15 +11,53 @@ importers when locating support scripts as well as when importing modules. import sys import imp +from pkgutil import read_code try: from imp import get_loader except ImportError: from pkgutil import get_loader __all__ = [ - "run_module", + "run_module", "run_path", ] +class _TempModule(object): + """Temporarily replace a module in sys.modules with an empty namespace""" + def __init__(self, mod_name): + self.mod_name = mod_name + self.module = imp.new_module(mod_name) + self._saved_module = [] + + def __enter__(self): + mod_name = self.mod_name + try: + self._saved_module.append(sys.modules[mod_name]) + except KeyError: + pass + sys.modules[mod_name] = self.module + return self + + def __exit__(self, *args): + if self._saved_module: + sys.modules[self.mod_name] = self._saved_module[0] + else: + del sys.modules[self.mod_name] + self._saved_module = [] + +class _ModifiedArgv0(object): + def __init__(self, value): + self.value = value + self._saved_value = self._sentinel = object() + + def __enter__(self): + if self._saved_value is not self._sentinel: + raise RuntimeError("Already preserving saved value") + self._saved_value = sys.argv[0] + sys.argv[0] = self.value + + def __exit__(self, *args): + self.value = self._sentinel + sys.argv[0] = self._saved_value def _run_code(code, run_globals, init_globals=None, mod_name=None, mod_fname=None, @@ -38,26 +76,10 @@ def _run_module_code(code, init_globals=None, mod_name=None, mod_fname=None, mod_loader=None, pkg_name=None): """Helper to run code in new namespace with sys modified""" - # Set up the top level namespace dictionary - temp_module = imp.new_module(mod_name) - mod_globals = temp_module.__dict__ - # Modify sys.argv[0] and sys.module[mod_name] - saved_argv0 = sys.argv[0] - restore_module = mod_name in sys.modules - if restore_module: - saved_module = sys.modules[mod_name] - sys.argv[0] = mod_fname - sys.modules[mod_name] = temp_module - try: + with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname): + mod_globals = temp_module.module.__dict__ _run_code(code, mod_globals, init_globals, - mod_name, mod_fname, - mod_loader, pkg_name) - finally: - sys.argv[0] = saved_argv0 - if restore_module: - sys.modules[mod_name] = saved_module - else: - del sys.modules[mod_name] + mod_name, mod_fname, mod_loader, pkg_name) # Copy the globals of the temporary module, as they # may be cleared when the temporary module goes away return mod_globals.copy() @@ -95,10 +117,22 @@ def _get_module_details(mod_name): return mod_name, loader, code, filename -# XXX ncoghlan: Should this be documented and made public? -# (Current thoughts: don't repeat the mistake that lead to its -# creation when run_module() no longer met the needs of -# mainmodule.c, but couldn't be changed because it was public) +def _get_main_module_details(): + # Helper that gives a nicer error message when attempting to + # execute a zipfile or directory by invoking __main__.py + main_name = "__main__" + try: + return _get_module_details(main_name) + except ImportError as exc: + if main_name in str(exc): + raise ImportError("can't find %r module in %r" % + (main_name, sys.path[0])) + raise + +# This function is the actual implementation of the -m switch and direct +# execution of zipfiles and directories and is deliberately kept private. +# This avoids a repeat of the situation where run_module() no longer met the +# needs of mainmodule.c, but couldn't be changed because it was public def _run_module_as_main(mod_name, alter_argv=True): """Runs the designated module in the __main__ namespace @@ -113,18 +147,12 @@ def _run_module_as_main(mod_name, alter_argv=True): __package__ """ try: - mod_name, loader, code, fname = _get_module_details(mod_name) + if alter_argv or mod_name != "__main__": # i.e. -m switch + mod_name, loader, code, fname = _get_module_details(mod_name) + else: # i.e. directory or zipfile execution + mod_name, loader, code, fname = _get_main_module_details() except ImportError as exc: - # Try to provide a good error message - # for directories, zip files and the -m switch - if alter_argv: - # For -m switch, just display the exception - info = str(exc) - else: - # For directories/zipfiles, let the user - # know what the code was looking for - info = "can't find '__main__.py' in %r" % sys.argv[0] - msg = "%s: %s" % (sys.executable, info) + msg = "%s: %s" % (sys.executable, str(exc)) sys.exit(msg) pkg_name = mod_name.rpartition('.')[0] main_globals = sys.modules["__main__"].__dict__ @@ -152,6 +180,95 @@ def run_module(mod_name, init_globals=None, fname, loader, pkg_name) +# XXX (ncoghlan): Perhaps expose the C API function +# as imp.get_importer instead of reimplementing it in Python? +def _get_importer(path_name): + """Python version of PyImport_GetImporter C API function""" + cache = sys.path_importer_cache + try: + importer = cache[path_name] + except KeyError: + # Not yet cached. Flag as using the + # standard machinery until we finish + # checking the hooks + cache[path_name] = None + for hook in sys.path_hooks: + try: + importer = hook(path_name) + break + except ImportError: + pass + else: + # The following check looks a bit odd. The trick is that + # NullImporter throws ImportError if the supplied path is a + # *valid* directory entry (and hence able to be handled + # by the standard import machinery) + try: + importer = imp.NullImporter(path_name) + except ImportError: + return None + cache[path_name] = importer + return importer + +def _get_code_from_file(fname): + # Check for a compiled file first + with open(fname, "rb") as f: + code = read_code(f) + if code is None: + # That didn't work, so try it as normal source code + with open(fname, "rU") as f: + code = compile(f.read(), fname, 'exec') + return code + +def run_path(path_name, init_globals=None, run_name=None): + """Execute code located at the specified filesystem location + + Returns the resulting top level namespace dictionary + + The file path may refer directly to a Python script (i.e. + one that could be directly executed with execfile) or else + it may refer to a zipfile or directory containing a top + level __main__.py script. + """ + if run_name is None: + run_name = "" + importer = _get_importer(path_name) + if isinstance(importer, imp.NullImporter): + # Not a valid sys.path entry, so run the code directly + # execfile() doesn't help as we want to allow compiled files + code = _get_code_from_file(path_name) + return _run_module_code(code, init_globals, run_name, path_name) + else: + # Importer is defined for path, so add it to + # the start of sys.path + sys.path.insert(0, path_name) + try: + # Here's where things are a little different from the run_module + # case. There, we only had to replace the module in sys while the + # code was running and doing so was somewhat optional. Here, we + # have no choice and we have to remove it even while we read the + # code. If we don't do this, a __loader__ attribute in the + # existing __main__ module may prevent location of the new module. + main_name = "__main__" + saved_main = sys.modules[main_name] + del sys.modules[main_name] + try: + mod_name, loader, code, fname = _get_main_module_details() + finally: + sys.modules[main_name] = saved_main + pkg_name = "" + with _TempModule(run_name) as temp_module, \ + _ModifiedArgv0(path_name): + mod_globals = temp_module.module.__dict__ + return _run_code(code, mod_globals, init_globals, + run_name, fname, loader, pkg_name) + finally: + try: + sys.path.remove(path_name) + except ValueError: + pass + + if __name__ == "__main__": # Run the module specified as the next command line argument if len(sys.argv) < 2: diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py new file mode 100644 index 00000000000..af5252c8c4f --- /dev/null +++ b/Lib/test/script_helper.py @@ -0,0 +1,119 @@ +# Common utility functions used by various script execution tests +# e.g. test_cmd_line, test_cmd_line_script and test_runpy + +import sys +import os +import os.path +import tempfile +import subprocess +import py_compile +import contextlib +import shutil +import zipfile + +# Executing the interpreter in a subprocess +def python_exit_code(*args): + cmd_line = [sys.executable, '-E'] + cmd_line.extend(args) + with open(os.devnull, 'w') as devnull: + return subprocess.call(cmd_line, stdout=devnull, + stderr=subprocess.STDOUT) + +def spawn_python(*args): + cmd_line = [sys.executable, '-E'] + cmd_line.extend(args) + return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + +def kill_python(p): + p.stdin.close() + data = p.stdout.read() + p.stdout.close() + # try to cleanup the child so we don't appear to leak when running + # with regrtest -R. This should be a no-op on Windows. + subprocess._cleanup() + return data + +def run_python(*args): + if __debug__: + p = spawn_python(*args) + else: + p = spawn_python('-O', *args) + stdout_data = kill_python(p) + return p.wait(), stdout_data + +# Script creation utilities +@contextlib.contextmanager +def temp_dir(): + dirname = tempfile.mkdtemp() + dirname = os.path.realpath(dirname) + try: + yield dirname + finally: + shutil.rmtree(dirname) + +def make_script(script_dir, script_basename, source): + script_filename = script_basename+os.extsep+'py' + script_name = os.path.join(script_dir, script_filename) + script_file = open(script_name, 'w') + script_file.write(source) + script_file.close() + return script_name + +def compile_script(script_name): + py_compile.compile(script_name, doraise=True) + if __debug__: + compiled_name = script_name + 'c' + else: + compiled_name = script_name + 'o' + return compiled_name + +def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + zip_filename = zip_basename+os.extsep+'zip' + zip_name = os.path.join(zip_dir, zip_filename) + zip_file = zipfile.ZipFile(zip_name, 'w') + if name_in_zip is None: + name_in_zip = os.path.basename(script_name) + zip_file.write(script_name, name_in_zip) + zip_file.close() + #if test.test_support.verbose: + # zip_file = zipfile.ZipFile(zip_name, 'r') + # print 'Contents of %r:' % zip_name + # zip_file.printdir() + # zip_file.close() + return zip_name, os.path.join(zip_name, name_in_zip) + +def make_pkg(pkg_dir): + os.mkdir(pkg_dir) + make_script(pkg_dir, '__init__', '') + +def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, + source, depth=1, compiled=False): + unlink = [] + init_name = make_script(zip_dir, '__init__', '') + unlink.append(init_name) + init_basename = os.path.basename(init_name) + script_name = make_script(zip_dir, script_basename, source) + unlink.append(script_name) + if compiled: + init_name = compile_script(init_name) + script_name = compile_script(script_name) + unlink.extend((init_name, script_name)) + pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)] + script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name)) + zip_filename = zip_basename+os.extsep+'zip' + zip_name = os.path.join(zip_dir, zip_filename) + zip_file = zipfile.ZipFile(zip_name, 'w') + for name in pkg_names: + init_name_in_zip = os.path.join(name, init_basename) + zip_file.write(init_name, init_name_in_zip) + zip_file.write(script_name, script_name_in_zip) + zip_file.close() + for name in unlink: + os.unlink(name) + #if test.test_support.verbose: + # zip_file = zipfile.ZipFile(zip_name, 'r') + # print 'Contents of %r:' % zip_name + # zip_file.printdir() + # zip_file.close() + return zip_name, os.path.join(zip_name, script_name_in_zip) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 09945f7e28c..424d19bff57 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,34 +5,16 @@ import os import test.test_support, unittest import sys -import subprocess +from test.script_helper import spawn_python, kill_python, python_exit_code -def _spawn_python(*args): - cmd_line = [sys.executable, '-E'] - cmd_line.extend(args) - return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - -def _kill_python(p): - p.stdin.close() - data = p.stdout.read() - p.stdout.close() - # try to cleanup the child so we don't appear to leak when running - # with regrtest -R. This should be a no-op on Windows. - subprocess._cleanup() - return data class CmdLineTest(unittest.TestCase): def start_python(self, *args): - p = _spawn_python(*args) - return _kill_python(p) + p = spawn_python(*args) + return kill_python(p) def exit_code(self, *args): - cmd_line = [sys.executable, '-E'] - cmd_line.extend(args) - with open(os.devnull, 'w') as devnull: - return subprocess.call(cmd_line, stdout=devnull, - stderr=subprocess.STDOUT) + return python_exit_code(*args) def test_directories(self): self.assertNotEqual(self.exit_code('.'), 0) @@ -85,10 +67,10 @@ class CmdLineTest(unittest.TestCase): # -m and -i need to play well together # Runs the timeit module and checks the __main__ # namespace has been populated appropriately - p = _spawn_python('-i', '-m', 'timeit', '-n', '1') + p = spawn_python('-i', '-m', 'timeit', '-n', '1') p.stdin.write('Timer\n') p.stdin.write('exit()\n') - data = _kill_python(p) + data = kill_python(p) self.assertTrue(data.startswith('1 loop')) self.assertTrue('__main__.Timer' in data) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index cefb9ae1575..bf1814ecbdd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -5,34 +5,12 @@ import os import os.path import sys import test.test_support -import tempfile -import subprocess -import py_compile -import contextlib -import shutil -import zipfile +from test.script_helper import (spawn_python, kill_python, run_python, + temp_dir, make_script, compile_script, + make_pkg, make_zip_script, make_zip_pkg) verbose = test.test_support.verbose -# XXX ncoghlan: Should we consider moving these to test_support? -from test_cmd_line import _spawn_python, _kill_python - -def _run_python(*args): - if __debug__: - p = _spawn_python(*args) - else: - p = _spawn_python('-O', *args) - stdout_data = _kill_python(p) - return p.wait(), stdout_data - -@contextlib.contextmanager -def temp_dir(): - dirname = tempfile.mkdtemp() - dirname = os.path.realpath(dirname) - try: - yield dirname - finally: - shutil.rmtree(dirname) test_source = """\ # Script may be run with optimisation enabled, so don't rely on assert @@ -60,63 +38,12 @@ print 'sys.argv[0]==%r' % sys.argv[0] """ def _make_test_script(script_dir, script_basename, source=test_source): - script_filename = script_basename+os.extsep+'py' - script_name = os.path.join(script_dir, script_filename) - script_file = open(script_name, 'w') - script_file.write(source) - script_file.close() - return script_name - -def _compile_test_script(script_name): - py_compile.compile(script_name, doraise=True) - if __debug__: - compiled_name = script_name + 'c' - else: - compiled_name = script_name + 'o' - return compiled_name - -def _make_test_zip(zip_dir, zip_basename, script_name, name_in_zip=None): - zip_filename = zip_basename+os.extsep+'zip' - zip_name = os.path.join(zip_dir, zip_filename) - zip_file = zipfile.ZipFile(zip_name, 'w') - if name_in_zip is None: - name_in_zip = os.path.basename(script_name) - zip_file.write(script_name, name_in_zip) - zip_file.close() - #if verbose: - # zip_file = zipfile.ZipFile(zip_name, 'r') - # print 'Contents of %r:' % zip_name - # zip_file.printdir() - # zip_file.close() - return zip_name, os.path.join(zip_name, name_in_zip) - -def _make_test_pkg(pkg_dir): - os.mkdir(pkg_dir) - _make_test_script(pkg_dir, '__init__', '') + return make_script(script_dir, script_basename, source) def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source=test_source, depth=1): - init_name = _make_test_script(zip_dir, '__init__', '') - init_basename = os.path.basename(init_name) - script_name = _make_test_script(zip_dir, script_basename, source) - pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)] - script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name)) - zip_filename = zip_basename+os.extsep+'zip' - zip_name = os.path.join(zip_dir, zip_filename) - zip_file = zipfile.ZipFile(zip_name, 'w') - for name in pkg_names: - init_name_in_zip = os.path.join(name, init_basename) - zip_file.write(init_name, init_name_in_zip) - zip_file.write(script_name, script_name_in_zip) - zip_file.close() - os.unlink(init_name) - os.unlink(script_name) - #if verbose: - # zip_file = zipfile.ZipFile(zip_name, 'r') - # print 'Contents of %r:' % zip_name - # zip_file.printdir() - # zip_file.close() - return zip_name, os.path.join(zip_name, script_name_in_zip) + return make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, + source, depth) # There's no easy way to pass the script directory in to get # -m to work (avoiding that is the whole point of making @@ -134,14 +61,14 @@ def _make_launch_script(script_dir, script_basename, module_name, path=None): else: path = repr(path) source = launch_source % (path, module_name) - return _make_test_script(script_dir, script_basename, source) + return make_script(script_dir, script_basename, source) class CmdLineTest(unittest.TestCase): def _check_script(self, script_name, expected_file, expected_argv0, expected_package, *cmd_line_switches): run_args = cmd_line_switches + (script_name,) - exit_code, data = _run_python(*run_args) + exit_code, data = run_python(*run_args) if verbose: print 'Output from test script %r:' % script_name print data @@ -161,7 +88,7 @@ class CmdLineTest(unittest.TestCase): def _check_import_error(self, script_name, expected_msg, *cmd_line_switches): run_args = cmd_line_switches + (script_name,) - exit_code, data = _run_python(*run_args) + exit_code, data = run_python(*run_args) if verbose: print 'Output from test script %r:' % script_name print data @@ -176,7 +103,7 @@ class CmdLineTest(unittest.TestCase): def test_script_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') - compiled_name = _compile_test_script(script_name) + compiled_name = compile_script(script_name) os.remove(script_name) self._check_script(compiled_name, compiled_name, compiled_name, None) @@ -188,39 +115,39 @@ class CmdLineTest(unittest.TestCase): def test_directory_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = _compile_test_script(script_name) + compiled_name = compile_script(script_name) os.remove(script_name) self._check_script(script_dir, compiled_name, script_dir, '') def test_directory_error(self): with temp_dir() as script_dir: - msg = "can't find '__main__.py' in %r" % script_dir + msg = "can't find '__main__' module in %r" % script_dir self._check_import_error(script_dir, msg) def test_zipfile(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - zip_name, run_name = _make_test_zip(script_dir, 'test_zip', script_name) + zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) self._check_script(zip_name, run_name, zip_name, '') def test_zipfile_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = _compile_test_script(script_name) - zip_name, run_name = _make_test_zip(script_dir, 'test_zip', compiled_name) + compiled_name = compile_script(script_name) + zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) self._check_script(zip_name, run_name, zip_name, '') def test_zipfile_error(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'not_main') - zip_name, run_name = _make_test_zip(script_dir, 'test_zip', script_name) - msg = "can't find '__main__.py' in %r" % zip_name + zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) + msg = "can't find '__main__' module in %r" % zip_name self._check_import_error(zip_name, msg) def test_module_in_package(self): with temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') - _make_test_pkg(pkg_dir) + make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, 'script') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') self._check_script(launch_name, script_name, script_name, 'test_pkg') @@ -240,7 +167,7 @@ class CmdLineTest(unittest.TestCase): def test_package(self): with temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') - _make_test_pkg(pkg_dir) + make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') self._check_script(launch_name, script_name, @@ -249,9 +176,9 @@ class CmdLineTest(unittest.TestCase): def test_package_compiled(self): with temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') - _make_test_pkg(pkg_dir) + make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') - compiled_name = _compile_test_script(script_name) + compiled_name = compile_script(script_name) os.remove(script_name) launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') self._check_script(launch_name, compiled_name, @@ -260,7 +187,7 @@ class CmdLineTest(unittest.TestCase): def test_package_error(self): with temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') - _make_test_pkg(pkg_dir) + make_pkg(pkg_dir) msg = ("'test_pkg' is a package and cannot " "be directly executed") launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') @@ -269,9 +196,9 @@ class CmdLineTest(unittest.TestCase): def test_package_recursion(self): with temp_dir() as script_dir: pkg_dir = os.path.join(script_dir, 'test_pkg') - _make_test_pkg(pkg_dir) + make_pkg(pkg_dir) main_dir = os.path.join(pkg_dir, '__main__') - _make_test_pkg(main_dir) + make_pkg(main_dir) msg = ("Cannot use package as __main__ module; " "'test_pkg' is a package and cannot " "be directly executed") diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 6cea41aab58..510bf1e7acf 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -5,8 +5,11 @@ import os.path import sys import tempfile from test.test_support import verbose, run_unittest, forget -from runpy import _run_code, _run_module_code, run_module +from test.script_helper import (temp_dir, make_script, compile_script, + make_pkg, make_zip_script, make_zip_pkg) + +from runpy import _run_code, _run_module_code, run_module, run_path # Note: This module can't safely test _run_module_as_main as it # runs its tests in the current process, which would mess with the # real __main__ module (usually test.regrtest) @@ -15,6 +18,7 @@ from runpy import _run_code, _run_module_code, run_module # Set up the test code and expected results class RunModuleCodeTest(unittest.TestCase): + """Unit tests for runpy._run_code and runpy._run_module_code""" expected_result = ["Top level assignment", "Lower level reference"] test_source = ( @@ -37,14 +41,14 @@ class RunModuleCodeTest(unittest.TestCase): def test_run_code(self): saved_argv0 = sys.argv[0] d = _run_code(self.test_source, {}) - self.assertTrue(d["result"] == self.expected_result) - self.assertTrue(d["__name__"] is None) - self.assertTrue(d["__file__"] is None) - self.assertTrue(d["__loader__"] is None) - self.assertTrue(d["__package__"] is None) - self.assertTrue(d["run_argv0"] is saved_argv0) - self.assertTrue("run_name" not in d) - self.assertTrue(sys.argv[0] is saved_argv0) + self.assertEqual(d["result"], self.expected_result) + self.assertIs(d["__name__"], None) + self.assertIs(d["__file__"], None) + self.assertIs(d["__loader__"], None) + self.assertIs(d["__package__"], None) + self.assertIs(d["run_argv0"], saved_argv0) + self.assertNotIn("run_name", d) + self.assertIs(sys.argv[0], saved_argv0) def test_run_module_code(self): initial = object() @@ -60,22 +64,23 @@ class RunModuleCodeTest(unittest.TestCase): file, loader, package) - self.assertTrue("result" not in d1) - self.assertTrue(d2["initial"] is initial) - self.assertTrue(d2["result"] == self.expected_result) - self.assertTrue(d2["nested"]["x"] == 1) - self.assertTrue(d2["__name__"] is name) + self.assertNotIn("result", d1) + self.assertIs(d2["initial"], initial) + self.assertEqual(d2["result"], self.expected_result) + self.assertEqual(d2["nested"]["x"], 1) + self.assertIs(d2["__name__"], name) self.assertTrue(d2["run_name_in_sys_modules"]) self.assertTrue(d2["module_in_sys_modules"]) - self.assertTrue(d2["__file__"] is file) - self.assertTrue(d2["run_argv0"] is file) - self.assertTrue(d2["__loader__"] is loader) - self.assertTrue(d2["__package__"] is package) - self.assertTrue(sys.argv[0] is saved_argv0) - self.assertTrue(name not in sys.modules) + self.assertIs(d2["__file__"], file) + self.assertIs(d2["run_argv0"], file) + self.assertIs(d2["__loader__"], loader) + self.assertIs(d2["__package__"], package) + self.assertIs(sys.argv[0], saved_argv0) + self.assertNotIn(name, sys.modules) class RunModuleTest(unittest.TestCase): + """Unit tests for runpy.run_module""" def expect_import_error(self, mod_name): try: @@ -272,9 +277,124 @@ from ..uncle.cousin import nephew self._check_relative_imports(depth, "__main__") +class RunPathTest(unittest.TestCase): + """Unit tests for runpy.run_path""" + # Based on corresponding tests in test_cmd_line_script + + test_source = """\ +# Script may be run with optimisation enabled, so don't rely on assert +# statements being executed +def assertEqual(lhs, rhs): + if lhs != rhs: + raise AssertionError('%r != %r' % (lhs, rhs)) +def assertIs(lhs, rhs): + if lhs is not rhs: + raise AssertionError('%r is not %r' % (lhs, rhs)) +# Check basic code execution +result = ['Top level assignment'] +def f(): + result.append('Lower level reference') +f() +assertEqual(result, ['Top level assignment', 'Lower level reference']) +# Check the sys module +import sys +assertIs(globals(), sys.modules[__name__].__dict__) +argv0 = sys.argv[0] +""" + + def _make_test_script(self, script_dir, script_basename, source=None): + if source is None: + source = self.test_source + return make_script(script_dir, script_basename, source) + + def _check_script(self, script_name, expected_name, expected_file, + expected_argv0, expected_package): + result = run_path(script_name) + self.assertEqual(result["__name__"], expected_name) + self.assertEqual(result["__file__"], expected_file) + self.assertIn("argv0", result) + self.assertEqual(result["argv0"], expected_argv0) + self.assertEqual(result["__package__"], expected_package) + + def _check_import_error(self, script_name, msg): + self.assertRaisesRegexp(ImportError, msg, run_path, script_name) + + def test_basic_script(self): + with temp_dir() as script_dir: + mod_name = 'script' + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(script_name, "", script_name, + script_name, None) + + def test_script_compiled(self): + with temp_dir() as script_dir: + mod_name = 'script' + script_name = self._make_test_script(script_dir, mod_name) + compiled_name = compile_script(script_name) + os.remove(script_name) + self._check_script(compiled_name, "", compiled_name, + compiled_name, None) + + def test_directory(self): + with temp_dir() as script_dir: + mod_name = '__main__' + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(script_dir, "", script_name, + script_dir, '') + + def test_directory_compiled(self): + with temp_dir() as script_dir: + mod_name = '__main__' + script_name = self._make_test_script(script_dir, mod_name) + compiled_name = compile_script(script_name) + os.remove(script_name) + self._check_script(script_dir, "", compiled_name, + script_dir, '') + + def test_directory_error(self): + with temp_dir() as script_dir: + mod_name = 'not_main' + script_name = self._make_test_script(script_dir, mod_name) + msg = "can't find '__main__' module in %r" % script_dir + self._check_import_error(script_dir, msg) + + def test_zipfile(self): + with temp_dir() as script_dir: + mod_name = '__main__' + script_name = self._make_test_script(script_dir, mod_name) + zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) + self._check_script(zip_name, "", fname, zip_name, '') + + def test_zipfile_compiled(self): + with temp_dir() as script_dir: + mod_name = '__main__' + script_name = self._make_test_script(script_dir, mod_name) + compiled_name = compile_script(script_name) + zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) + self._check_script(zip_name, "", fname, zip_name, '') + + def test_zipfile_error(self): + with temp_dir() as script_dir: + mod_name = 'not_main' + script_name = self._make_test_script(script_dir, mod_name) + zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) + msg = "can't find '__main__' module in %r" % zip_name + self._check_import_error(zip_name, msg) + + def test_main_recursion_error(self): + with temp_dir() as script_dir, temp_dir() as dummy_dir: + mod_name = '__main__' + source = ("import runpy\n" + "runpy.run_path(%r)\n") % dummy_dir + script_name = self._make_test_script(script_dir, mod_name, source) + zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) + msg = "recursion depth exceeded" + self.assertRaisesRegexp(RuntimeError, msg, run_path, zip_name) + + + def test_main(): - run_unittest(RunModuleCodeTest) - run_unittest(RunModuleTest) + run_unittest(RunModuleCodeTest, RunModuleTest, RunPathTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 1f0278731f5..1b641388843 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -14,6 +14,9 @@ import doctest import inspect import linecache import pdb +from test.script_helper import (spawn_python, kill_python, run_python, + temp_dir, make_script, compile_script, + make_pkg, make_zip_script, make_zip_pkg) verbose = test.test_support.verbose @@ -29,11 +32,6 @@ verbose = test.test_support.verbose # Retrieve some helpers from other test cases from test import test_doctest, sample_doctest from test.test_importhooks import ImportHooksBaseTestCase -from test.test_cmd_line_script import temp_dir, _run_python, \ - _spawn_python, _kill_python, \ - _make_test_script, \ - _compile_test_script, \ - _make_test_zip, _make_test_pkg def _run_object_doctest(obj, module): @@ -78,10 +76,10 @@ class ZipSupportTests(ImportHooksBaseTestCase): def test_inspect_getsource_issue4223(self): test_src = "def foo(): pass\n" with temp_dir() as d: - init_name = _make_test_script(d, '__init__', test_src) + init_name = make_script(d, '__init__', test_src) name_in_zip = os.path.join('zip_pkg', os.path.basename(init_name)) - zip_name, run_name = _make_test_zip(d, 'test_zip', + zip_name, run_name = make_zip_script(d, 'test_zip', init_name, name_in_zip) os.remove(init_name) sys.path.insert(0, zip_name) @@ -106,9 +104,9 @@ class ZipSupportTests(ImportHooksBaseTestCase): sample_src = sample_src.replace("test.test_doctest", "test_zipped_doctest") with temp_dir() as d: - script_name = _make_test_script(d, 'test_zipped_doctest', + script_name = make_script(d, 'test_zipped_doctest', test_src) - zip_name, run_name = _make_test_zip(d, 'test_zip', + zip_name, run_name = make_zip_script(d, 'test_zip', script_name) z = zipfile.ZipFile(zip_name, 'a') z.writestr("sample_zipped_doctest.py", sample_src) @@ -184,17 +182,17 @@ class ZipSupportTests(ImportHooksBaseTestCase): """) pattern = 'File "%s", line 2, in %s' with temp_dir() as d: - script_name = _make_test_script(d, 'script', test_src) - exit_code, data = _run_python(script_name) + script_name = make_script(d, 'script', test_src) + exit_code, data = run_python(script_name) expected = pattern % (script_name, "__main__.Test") if verbose: print "Expected line", expected print "Got stdout:" print data self.assertTrue(expected in data) - zip_name, run_name = _make_test_zip(d, "test_zip", + zip_name, run_name = make_zip_script(d, "test_zip", script_name, '__main__.py') - exit_code, data = _run_python(zip_name) + exit_code, data = run_python(zip_name) expected = pattern % (run_name, "__main__.Test") if verbose: print "Expected line", expected @@ -211,16 +209,16 @@ class ZipSupportTests(ImportHooksBaseTestCase): pdb.runcall(f) """) with temp_dir() as d: - script_name = _make_test_script(d, 'script', test_src) - p = _spawn_python(script_name) + script_name = make_script(d, 'script', test_src) + p = spawn_python(script_name) p.stdin.write('l\n') - data = _kill_python(p) + data = kill_python(p) self.assertTrue(script_name in data) - zip_name, run_name = _make_test_zip(d, "test_zip", + zip_name, run_name = make_zip_script(d, "test_zip", script_name, '__main__.py') - p = _spawn_python(zip_name) + p = spawn_python(zip_name) p.stdin.write('l\n') - data = _kill_python(p) + data = kill_python(p) self.assertTrue(run_name in data) diff --git a/Misc/NEWS b/Misc/NEWS index afa286de6ef..3feccda6f66 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -429,6 +429,12 @@ Core and Builtins Library ------- +- Issue #6816: runpy now provides a run_path function that allows Python code + to execute file paths that refer to source or compiled Python files as well + as zipfiles, directories and other valid sys.path entries that contain a + __main__.py file. This allows applications that run other Python scripts to + support the same flexibility as the CPython command line itself. + - Issue #7318: multiprocessing now uses a timeout when it fails to establish a connection with another process, rather than looping endlessly. The default timeout is 20 seconds, which should be amply sufficient for