diff --git a/Lib/bdb.py b/Lib/bdb.py index 88088a51969..6dc54eda4bb 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -346,7 +346,7 @@ class Bdb: rv = frame.f_locals['__return__'] s = s + '->' s = s + reprlib.repr(rv) - line = linecache.getline(filename, lineno) + line = linecache.getline(filename, lineno, frame.f_globals) if line: s = s + lprefix + line.strip() return s @@ -588,7 +588,7 @@ class Tdb(Bdb): name = frame.f_code.co_name if not name: name = '???' fn = self.canonic(frame.f_code.co_filename) - line = linecache.getline(fn, frame.f_lineno) + line = linecache.getline(fn, frame.f_lineno, frame.f_globals) print('+++', fn, frame.f_lineno, name, ':', line.strip()) def user_return(self, frame, retval): print('+++ return', retval) diff --git a/Lib/linecache.py b/Lib/linecache.py index 50a0c1b480c..6a9535ed1f4 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -89,21 +89,20 @@ def updatecache(filename, module_globals=None): get_source = getattr(loader, 'get_source', None) if name and get_source: - if basename.startswith(name.split('.')[-1]+'.'): - try: - data = get_source(name) - except (ImportError, IOError): - pass - else: - if data is None: - # No luck, the PEP302 loader cannot find the source - # for this module. - return [] - cache[filename] = ( - len(data), None, - [line+'\n' for line in data.splitlines()], fullname - ) - return cache[filename][2] + try: + data = get_source(name) + except (ImportError, IOError): + pass + else: + if data is None: + # No luck, the PEP302 loader cannot find the source + # for this module. + return [] + cache[filename] = ( + len(data), None, + [line+'\n' for line in data.splitlines()], fullname + ) + return cache[filename][2] # Try looking through the module search path. diff --git a/Lib/pdb.py b/Lib/pdb.py index e697dfab4ba..ca03c505a65 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -438,7 +438,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank line or EOF). Warning: testing is not comprehensive. """ - line = linecache.getline(filename, lineno) + line = linecache.getline(filename, lineno, self.curframe.f_globals) if not line: print('End of file', file=self.stdout) return 0 @@ -768,7 +768,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): breaklist = self.get_file_breaks(filename) try: for lineno in range(first, last+1): - line = linecache.getline(filename, lineno) + line = linecache.getline(filename, lineno, self.curframe.f_globals) if not line: print('[EOF]', file=self.stdout) break diff --git a/Lib/runpy.py b/Lib/runpy.py index 5038266ff33..22a29896d88 100755 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -65,13 +65,14 @@ def _run_module_code(code, init_globals=None, # This helper is needed due to a missing component in the PEP 302 # loader protocol (specifically, "get_filename" is non-standard) +# Since we can't introduce new features in maintenance releases, +# support was added to zipimporter under the name '_get_filename' def _get_filename(loader, mod_name): - try: - get_filename = loader.get_filename - except AttributeError: - return None - else: - return get_filename(mod_name) + for attr in ("get_filename", "_get_filename"): + meth = getattr(loader, attr, None) + if meth is not None: + return meth(mod_name) + return None # Helper to get the loader, code and filename for a module def _get_module_details(mod_name): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e0b09ea8ee5..7fab3832645 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -75,36 +75,66 @@ def _compile_test_script(script_name): compiled_name = script_name + 'o' return compiled_name -def _make_test_zip(zip_dir, zip_basename, script_name): +def _make_test_zip(zip_dir, zip_basename, script_name, name_in_zip=None): zip_filename = zip_basename+os.path.extsep+"zip" zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') - zip_file.write(script_name, os.path.basename(script_name)) + if name_in_zip is None: + name_in_zip = os.path.basename(script_name) + zip_file.write(script_name, name_in_zip) zip_file.close() - # if verbose: + #if verbose: # zip_file = zipfile.ZipFile(zip_name, 'r') # print("Contents of %r:" % zip_name) # zip_file.printdir() # zip_file.close() - return zip_name + return zip_name, os.path.join(zip_name, name_in_zip) def _make_test_pkg(pkg_dir): os.mkdir(pkg_dir) _make_test_script(pkg_dir, '__init__', '') +def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, + source=test_source, depth=1): + init_name = _make_test_script(zip_dir, '__init__', '') + init_basename = os.path.basename(init_name) + script_name = _make_test_script(zip_dir, script_basename, source) + pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)] + script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name)) + zip_filename = zip_basename+os.extsep+'zip' + zip_name = os.path.join(zip_dir, zip_filename) + zip_file = zipfile.ZipFile(zip_name, 'w') + for name in pkg_names: + init_name_in_zip = os.path.join(name, init_basename) + zip_file.write(init_name, init_name_in_zip) + zip_file.write(script_name, script_name_in_zip) + zip_file.close() + os.unlink(init_name) + os.unlink(script_name) + #if verbose: + # zip_file = zipfile.ZipFile(zip_name, 'r') + # print 'Contents of %r:' % zip_name + # zip_file.printdir() + # zip_file.close() + return zip_name, os.path.join(zip_name, script_name_in_zip) + # There's no easy way to pass the script directory in to get # -m to work (avoiding that is the whole point of making # directories and zipfiles executable!) # So we fake it for testing purposes with a custom launch script launch_source = """\ import sys, os.path, runpy -sys.path[0:0] = os.path.dirname(__file__) +sys.path.insert(0, %s) runpy._run_module_as_main(%r) """ -def _make_launch_script(script_dir, script_basename, module_name): - return _make_test_script(script_dir, script_basename, - launch_source % module_name) +def _make_launch_script(script_dir, script_basename, module_name, path=None): + if path is None: + path = "os.path.dirname(__file__)" + else: + path = repr(path) + source = launch_source % (path, module_name) + return _make_test_script(script_dir, script_basename, source) class CmdLineTest(unittest.TestCase): def _check_script(self, script_name, expected_file, @@ -155,15 +185,15 @@ class CmdLineTest(unittest.TestCase): def test_zipfile(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - zip_name = _make_test_zip(script_dir, 'test_zip', script_name) - self._check_script(zip_name, None, zip_name, '') + zip_name, run_name = _make_test_zip(script_dir, 'test_zip', script_name) + self._check_script(zip_name, run_name, zip_name, '') def test_zipfile_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') compiled_name = _compile_test_script(script_name) - zip_name = _make_test_zip(script_dir, 'test_zip', compiled_name) - self._check_script(zip_name, None, zip_name, '') + zip_name, run_name = _make_test_zip(script_dir, 'test_zip', compiled_name) + self._check_script(zip_name, run_name, zip_name, '') def test_module_in_package(self): with temp_dir() as script_dir: @@ -171,8 +201,19 @@ class CmdLineTest(unittest.TestCase): _make_test_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, 'script') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') - self._check_script(launch_name, script_name, - script_name, 'test_pkg') + self._check_script(launch_name, script_name, script_name, 'test_pkg') + + def test_module_in_package_in_zipfile(self): + with temp_dir() as script_dir: + zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) + self._check_script(launch_name, run_name, run_name, 'test_pkg') + + def test_module_in_subpackage_in_zipfile(self): + with temp_dir() as script_dir: + zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) + self._check_script(launch_name, run_name, run_name, 'test_pkg.test_pkg') def test_main(): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index f07da75011d..9a128fb2257 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -6,6 +6,9 @@ from test import support import doctest import warnings +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. + ###################################################################### ## Sample Objects (used by test cases) ###################################################################### @@ -369,7 +372,7 @@ We'll simulate a __file__ attr that ends in pyc: >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 7b856162052..bde6d6c83c7 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -18,6 +18,9 @@ from test import inspect_fodder2 as mod2 # getclasstree, getargspec, getargvalues, formatargspec, formatargvalues, # currentframe, stack, trace, isdatadescriptor +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. + modfile = mod.__file__ if modfile.endswith(('c', 'o')): modfile = modfile[:-1] diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 754476a6955..5637467319f 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -211,16 +211,24 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): zi = zipimport.zipimporter(TEMP_ZIP) self.assertEquals(zi.archive, TEMP_ZIP) self.assertEquals(zi.is_package(TESTPACK), True) - zi.load_module(TESTPACK) + mod = zi.load_module(TESTPACK) + self.assertEquals(zi._get_filename(TESTPACK), mod.__file__) self.assertEquals(zi.is_package(packdir + '__init__'), False) self.assertEquals(zi.is_package(packdir + TESTPACK2), True) self.assertEquals(zi.is_package(packdir2 + TESTMOD), False) - mod_name = packdir2 + TESTMOD - mod = __import__(module_path_to_dotted_name(mod_name)) + mod_path = packdir2 + TESTMOD + mod_name = module_path_to_dotted_name(mod_path) + pkg = __import__(mod_name) + mod = sys.modules[mod_name] self.assertEquals(zi.get_source(TESTPACK), None) - self.assertEquals(zi.get_source(mod_name), None) + self.assertEquals(zi.get_source(mod_path), None) + self.assertEquals(zi._get_filename(mod_path), mod.__file__) + # To pass in the module name instead of the path, we must use the right importer + loader = mod.__loader__ + self.assertEquals(loader.get_source(mod_name), None) + self.assertEquals(loader._get_filename(mod_name), mod.__file__) # test prefix and archivepath members zi2 = zipimport.zipimporter(TEMP_ZIP + os.sep + TESTPACK) @@ -248,15 +256,23 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEquals(zi.archive, TEMP_ZIP) self.assertEquals(zi.prefix, packdir) self.assertEquals(zi.is_package(TESTPACK2), True) - zi.load_module(TESTPACK2) + mod = zi.load_module(TESTPACK2) + self.assertEquals(zi._get_filename(TESTPACK2), mod.__file__) self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False) self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) - mod_name = TESTPACK2 + os.sep + TESTMOD - mod = __import__(module_path_to_dotted_name(mod_name)) + mod_path = TESTPACK2 + os.sep + TESTMOD + mod_name = module_path_to_dotted_name(mod_path) + pkg = __import__(mod_name) + mod = sys.modules[mod_name] self.assertEquals(zi.get_source(TESTPACK2), None) - self.assertEquals(zi.get_source(mod_name), None) + self.assertEquals(zi.get_source(mod_path), None) + self.assertEquals(zi._get_filename(mod_path), mod.__file__) + # To pass in the module name instead of the path, we must use the right importer + loader = mod.__loader__ + self.assertEquals(loader.get_source(mod_name), None) + self.assertEquals(loader._get_filename(mod_name), mod.__file__) finally: z.close() os.remove(TEMP_ZIP) diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py new file mode 100644 index 00000000000..7942e37ff2c --- /dev/null +++ b/Lib/test/test_zipimport_support.py @@ -0,0 +1,199 @@ +# This test module covers support in various parts of the standard library +# for working with modules located inside zipfiles +# The tests are centralised in this fashion to make it easy to drop them +# if a platform doesn't support zipimport +import unittest +import test.support +import os +import os.path +import sys +import textwrap +import zipfile +import zipimport +import doctest +import inspect +import linecache +import pdb + +verbose = test.support.verbose + +# Library modules covered by this test set +# pdb (Issue 4201) +# inspect (Issue 4223) +# doctest (Issue 4197) + +# Other test modules with zipimport related tests +# test_zipimport (of course!) +# test_cmd_line_script (covers the zipimport support in runpy) + +# Retrieve some helpers from other test cases +from test import test_doctest, sample_doctest +from test.test_importhooks import ImportHooksBaseTestCase +from test.test_cmd_line_script import temp_dir, _run_python, \ + _spawn_python, _kill_python, \ + _make_test_script, \ + _compile_test_script, \ + _make_test_zip, _make_test_pkg + + +def _run_object_doctest(obj, module): + # Direct doctest output (normally just errors) to real stdout; doctest + # output shouldn't be compared by regrtest. + save_stdout = sys.stdout + sys.stdout = test.support.get_original_stdout() + try: + finder = doctest.DocTestFinder(verbose=verbose, recurse=False) + runner = doctest.DocTestRunner(verbose=verbose) + # Use the object's fully qualified name if it has one + # Otherwise, use the module's name + try: + name = "%s.%s" % (obj.__module__, obj.__name__) + except AttributeError: + name = module.__name__ + for example in finder.find(obj, name, module): + runner.run(example) + f, t = runner.failures, runner.tries + if f: + raise test.support.TestFailed("%d of %d doctests failed" % (f, t)) + finally: + sys.stdout = save_stdout + if verbose: + print ('doctest (%s) ... %d tests with zero failures' % (module.__name__, t)) + return f, t + + + +class ZipSupportTests(ImportHooksBaseTestCase): + # We use the ImportHooksBaseTestCase to restore + # the state of the import related information + # in the sys module after each test + # We also clear the linecache and zipimport cache + # just to avoid any bogus errors due to name reuse in the tests + def setUp(self): + linecache.clearcache() + zipimport._zip_directory_cache.clear() + ImportHooksBaseTestCase.setUp(self) + + + def test_inspect_getsource_issue4223(self): + test_src = "def foo(): pass\n" + with temp_dir() as d: + init_name = _make_test_script(d, '__init__', test_src) + name_in_zip = os.path.join('zip_pkg', + os.path.basename(init_name)) + zip_name, run_name = _make_test_zip(d, 'test_zip', + init_name, name_in_zip) + os.remove(init_name) + sys.path.insert(0, zip_name) + import zip_pkg + self.assertEqual(inspect.getsource(zip_pkg.foo), test_src) + + def test_doctest_issue4197(self): + # To avoid having to keep two copies of the doctest module's + # unit tests in sync, this test works by taking the source of + # test_doctest itself, rewriting it a bit to cope with a new + # location, and then throwing it in a zip file to make sure + # everything still works correctly + test_src = inspect.getsource(test_doctest) + test_src = test_src.replace( + "from test import test_doctest", + "import test_zipped_doctest as test_doctest") + test_src = test_src.replace("test.test_doctest", + "test_zipped_doctest") + test_src = test_src.replace("test.sample_doctest", + "sample_zipped_doctest") + sample_src = inspect.getsource(sample_doctest) + sample_src = sample_src.replace("test.test_doctest", + "test_zipped_doctest") + with temp_dir() as d: + script_name = _make_test_script(d, 'test_zipped_doctest', + test_src) + zip_name, run_name = _make_test_zip(d, 'test_zip', + script_name) + z = zipfile.ZipFile(zip_name, 'a') + z.writestr("sample_zipped_doctest.py", sample_src) + z.close() + if verbose: + zip_file = zipfile.ZipFile(zip_name, 'r') + print ('Contents of %r:' % zip_name) + zip_file.printdir() + zip_file.close() + os.remove(script_name) + sys.path.insert(0, zip_name) + import test_zipped_doctest + # Some of the doc tests depend on the colocated text files + # which aren't available to the zipped version (the doctest + # module currently requires real filenames for non-embedded + # tests). So we're forced to be selective about which tests + # to run. + # doctest could really use some APIs which take a text + # string or a file object instead of a filename... + known_good_tests = [ + test_zipped_doctest.SampleClass, + test_zipped_doctest.SampleClass.NestedClass, + test_zipped_doctest.SampleClass.NestedClass.__init__, + test_zipped_doctest.SampleClass.__init__, + test_zipped_doctest.SampleClass.a_classmethod, + test_zipped_doctest.SampleClass.a_property, + test_zipped_doctest.SampleClass.a_staticmethod, + test_zipped_doctest.SampleClass.double, + test_zipped_doctest.SampleClass.get, + test_zipped_doctest.SampleNewStyleClass, + test_zipped_doctest.SampleNewStyleClass.__init__, + test_zipped_doctest.SampleNewStyleClass.double, + test_zipped_doctest.SampleNewStyleClass.get, + test_zipped_doctest.sample_func, + test_zipped_doctest.test_DocTest, + test_zipped_doctest.test_DocTestParser, + test_zipped_doctest.test_DocTestRunner.basics, + test_zipped_doctest.test_DocTestRunner.exceptions, + test_zipped_doctest.test_DocTestRunner.option_directives, + test_zipped_doctest.test_DocTestRunner.optionflags, + test_zipped_doctest.test_DocTestRunner.verbose_flag, + test_zipped_doctest.test_Example, + test_zipped_doctest.test_debug, + test_zipped_doctest.test_pdb_set_trace, + test_zipped_doctest.test_pdb_set_trace_nested, + test_zipped_doctest.test_testsource, + test_zipped_doctest.test_trailing_space_in_test, + test_zipped_doctest.test_DocTestSuite, + test_zipped_doctest.test_DocTestFinder, + ] + # These remaining tests are the ones which need access + # to the data files, so we don't run them + fail_due_to_missing_data_files = [ + test_zipped_doctest.test_DocFileSuite, + test_zipped_doctest.test_testfile, + test_zipped_doctest.test_unittest_reportflags, + ] + for obj in known_good_tests: + _run_object_doctest(obj, test_zipped_doctest) + + def test_pdb_issue4201(self): + test_src = textwrap.dedent("""\ + def f(): + pass + + import pdb + pdb.runcall(f) + """) + with temp_dir() as d: + script_name = _make_test_script(d, 'script', test_src) + p = _spawn_python(script_name) + p.stdin.write(b'l\n') + data = _kill_python(p).decode() + self.assert_(script_name in data) + zip_name, run_name = _make_test_zip(d, "test_zip", + script_name, '__main__.py') + p = _spawn_python(zip_name) + p.stdin.write(b'l\n') + data = _kill_python(p).decode() + self.assert_(run_name in data) + + +def test_main(): + test.support.run_unittest(ZipSupportTests) + test.support.reap_children() + +if __name__ == '__main__': + test_main() diff --git a/Misc/NEWS b/Misc/NEWS index e31bb0b8101..54ae4712355 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -45,6 +45,25 @@ Core and Builtins Library ------- +- Issue #4223: inspect.getsource() will now correctly display source code + for packages loaded via zipimport (or any other conformant PEP 302 + loader). Original patch by Alexander Belopolsky. + +- Issue #4201: pdb can now access and display source code loaded via + zipimport (or any other conformant PEP 302 loader). Original patch by + Alexander Belopolsky. + +- Issue #4197: doctests in modules loaded via zipimport (or any other PEP + 302 conformant loader) will now work correctly in most cases (they + are still subject to the constraints that exist for all code running + from inside a module loaded via a PEP 302 loader and attempting to + perform IO operations based on __file__). Original patch by + Alexander Belopolsky. + +- Issues #4082 and #4512: Add runpy support to zipimport in a manner that + allows backporting to maintenance branches. Original patch by + Alexander Belopolsky. + - Issue #4163: textwrap module: allow word splitting on a hyphen preceded by a non-ASCII letter. diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 000e9874e25..ae11bdeeb44 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -354,6 +354,29 @@ error: return NULL; } +/* Return a string matching __file__ for the named module */ +static PyObject * +zipimporter_get_filename(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + PyObject *code; + char *fullname, *modpath; + int ispackage; + + if (!PyArg_ParseTuple(args, "s:zipimporter._get_filename", + &fullname)) + return NULL; + + /* Deciding the filename requires working out where the code + would come from if the module was actually loaded */ + code = get_module_code(self, fullname, &ispackage, &modpath); + if (code == NULL) + return NULL; + Py_DECREF(code); /* Only need the path info */ + + return PyUnicode_FromString(modpath); +} + /* Return a bool signifying whether the module is a package or not. */ static PyObject * zipimporter_is_package(PyObject *obj, PyObject *args) @@ -518,6 +541,12 @@ Return the source code for the specified module. Raise ZipImportError\n\ is the module couldn't be found, return None if the archive does\n\ contain the module, but has no source for it."); + +PyDoc_STRVAR(doc_get_filename, +"_get_filename(fullname) -> filename string.\n\ +\n\ +Return the filename for the specified module."); + static PyMethodDef zipimporter_methods[] = { {"find_module", zipimporter_find_module, METH_VARARGS, doc_find_module}, @@ -529,6 +558,8 @@ static PyMethodDef zipimporter_methods[] = { doc_get_code}, {"get_source", zipimporter_get_source, METH_VARARGS, doc_get_source}, + {"_get_filename", zipimporter_get_filename, METH_VARARGS, + doc_get_filename}, {"is_package", zipimporter_is_package, METH_VARARGS, doc_is_package}, {NULL, NULL} /* sentinel */