traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merged revisions 67750-67751 via svnmerge from 

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r67750 | nick.coghlan | 2008-12-14 20:54:50 +1000 (Sun, 14 Dec 2008) | 1 line

  Fix several issues relating to access to source code inside zipfiles. Initial work by Alexander Belopolsky. See Misc/NEWS in this checkin for details.
........
  r67751 | nick.coghlan | 2008-12-14 21:09:40 +1000 (Sun, 14 Dec 2008) | 1 line

  Add file that was missed from r67750
........
This commit is contained in:
Nick Coghlan 2008-12-14 11:50:48 +00:00
parent 80a0c7f623
commit f088e5e6cc
11 changed files with 360 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Lib/bdb.py
View File

@ -346,7 +346,7 @@ class Bdb:
rv = frame.f_locals['__return__']
s = s + '->'
s = s + reprlib.repr(rv)
line = linecache.getline(filename, lineno)
line = linecache.getline(filename, lineno, frame.f_globals)
if line: s = s + lprefix + line.strip()
return s
@ -588,7 +588,7 @@ class Tdb(Bdb):
name = frame.f_code.co_name
if not name: name = '???'
fn = self.canonic(frame.f_code.co_filename)
line = linecache.getline(fn, frame.f_lineno)
line = linecache.getline(fn, frame.f_lineno, frame.f_globals)
print('+++', fn, frame.f_lineno, name, ':', line.strip())
def user_return(self, frame, retval):
print('+++ return', retval)

29
Lib/linecache.py
View File

@ -89,21 +89,20 @@ def updatecache(filename, module_globals=None):
get_source = getattr(loader, 'get_source', None)
if name and get_source:
if basename.startswith(name.split('.')[-1]+'.'):
try:
data = get_source(name)
except (ImportError, IOError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (
len(data), None,
[line+'\n' for line in data.splitlines()], fullname
)
return cache[filename][2]
try:
data = get_source(name)
except (ImportError, IOError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (
len(data), None,
[line+'\n' for line in data.splitlines()], fullname
)
return cache[filename][2]
# Try looking through the module search path.

4
Lib/pdb.py
View File

@ -438,7 +438,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
line or EOF). Warning: testing is not comprehensive.
"""
line = linecache.getline(filename, lineno)
line = linecache.getline(filename, lineno, self.curframe.f_globals)
if not line:
print('End of file', file=self.stdout)
return 0
@ -768,7 +768,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
breaklist = self.get_file_breaks(filename)
try:
for lineno in range(first, last+1):
line = linecache.getline(filename, lineno)
line = linecache.getline(filename, lineno, self.curframe.f_globals)
if not line:
print('[EOF]', file=self.stdout)
break

13
Lib/runpy.py
View File

@ -65,13 +65,14 @@ def _run_module_code(code, init_globals=None,
# This helper is needed due to a missing component in the PEP 302
# loader protocol (specifically, "get_filename" is non-standard)
# Since we can't introduce new features in maintenance releases,
# support was added to zipimporter under the name '_get_filename'
def _get_filename(loader, mod_name):
try:
get_filename = loader.get_filename
except AttributeError:
return None
else:
return get_filename(mod_name)
for attr in ("get_filename", "_get_filename"):
meth = getattr(loader, attr, None)
if meth is not None:
return meth(mod_name)
return None
# Helper to get the loader, code and filename for a module
def _get_module_details(mod_name):

69
Lib/test/test_cmd_line_script.py
View File

@ -75,36 +75,66 @@ def _compile_test_script(script_name):
compiled_name = script_name + 'o'
return compiled_name
def _make_test_zip(zip_dir, zip_basename, script_name):
def _make_test_zip(zip_dir, zip_basename, script_name, name_in_zip=None):
zip_filename = zip_basename+os.path.extsep+"zip"
zip_name = os.path.join(zip_dir, zip_filename)
zip_file = zipfile.ZipFile(zip_name, 'w')
zip_file.write(script_name, os.path.basename(script_name))
if name_in_zip is None:
name_in_zip = os.path.basename(script_name)
zip_file.write(script_name, name_in_zip)
zip_file.close()
# if verbose:
#if verbose:
# zip_file = zipfile.ZipFile(zip_name, 'r')
# print("Contents of %r:" % zip_name)
# zip_file.printdir()
# zip_file.close()
return zip_name
return zip_name, os.path.join(zip_name, name_in_zip)
def _make_test_pkg(pkg_dir):
os.mkdir(pkg_dir)
_make_test_script(pkg_dir, '__init__', '')
def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
source=test_source, depth=1):
init_name = _make_test_script(zip_dir, '__init__', '')
init_basename = os.path.basename(init_name)
script_name = _make_test_script(zip_dir, script_basename, source)
pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
zip_filename = zip_basename+os.extsep+'zip'
zip_name = os.path.join(zip_dir, zip_filename)
zip_file = zipfile.ZipFile(zip_name, 'w')
for name in pkg_names:
init_name_in_zip = os.path.join(name, init_basename)
zip_file.write(init_name, init_name_in_zip)
zip_file.write(script_name, script_name_in_zip)
zip_file.close()
os.unlink(init_name)
os.unlink(script_name)
#if verbose:
# zip_file = zipfile.ZipFile(zip_name, 'r')
# print 'Contents of %r:' % zip_name
# zip_file.printdir()
# zip_file.close()
return zip_name, os.path.join(zip_name, script_name_in_zip)
# There's no easy way to pass the script directory in to get
# -m to work (avoiding that is the whole point of making
# directories and zipfiles executable!)
# So we fake it for testing purposes with a custom launch script
launch_source = """\
import sys, os.path, runpy
sys.path[0:0] = os.path.dirname(__file__)
sys.path.insert(0, %s)
runpy._run_module_as_main(%r)
"""
def _make_launch_script(script_dir, script_basename, module_name):
return _make_test_script(script_dir, script_basename,
launch_source % module_name)
def _make_launch_script(script_dir, script_basename, module_name, path=None):
if path is None:
path = "os.path.dirname(__file__)"
else:
path = repr(path)
source = launch_source % (path, module_name)
return _make_test_script(script_dir, script_basename, source)
class CmdLineTest(unittest.TestCase):
def _check_script(self, script_name, expected_file,
@ -155,15 +185,15 @@ class CmdLineTest(unittest.TestCase):
def test_zipfile(self):
with temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
zip_name = _make_test_zip(script_dir, 'test_zip', script_name)
self._check_script(zip_name, None, zip_name, '')
zip_name, run_name = _make_test_zip(script_dir, 'test_zip', script_name)
self._check_script(zip_name, run_name, zip_name, '')
def test_zipfile_compiled(self):
with temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
compiled_name = _compile_test_script(script_name)
zip_name = _make_test_zip(script_dir, 'test_zip', compiled_name)
self._check_script(zip_name, None, zip_name, '')
zip_name, run_name = _make_test_zip(script_dir, 'test_zip', compiled_name)
self._check_script(zip_name, run_name, zip_name, '')
def test_module_in_package(self):
with temp_dir() as script_dir:
@ -171,8 +201,19 @@ class CmdLineTest(unittest.TestCase):
_make_test_pkg(pkg_dir)
script_name = _make_test_script(pkg_dir, 'script')
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script')
self._check_script(launch_name, script_name,
script_name, 'test_pkg')
self._check_script(launch_name, script_name, script_name, 'test_pkg')
def test_module_in_package_in_zipfile(self):
with temp_dir() as script_dir:
zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
self._check_script(launch_name, run_name, run_name, 'test_pkg')
def test_module_in_subpackage_in_zipfile(self):
with temp_dir() as script_dir:
zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
self._check_script(launch_name, run_name, run_name, 'test_pkg.test_pkg')
def test_main():

5
Lib/test/test_doctest.py
View File

@ -6,6 +6,9 @@ from test import support
import doctest
import warnings
# NOTE: There are some additional tests relating to interaction with
# zipimport in the test_zipimport_support test module.
######################################################################
## Sample Objects (used by test cases)
######################################################################
@ -369,7 +372,7 @@ We'll simulate a __file__ attr that ends in pyc:
>>> tests = finder.find(sample_func)
>>> print(tests) # doctest: +ELLIPSIS
[<DocTest sample_func from ...:13 (1 example)>]
[<DocTest sample_func from ...:16 (1 example)>]
The exact name depends on how test_doctest was invoked, so allow for
leading path components.

3
Lib/test/test_inspect.py
View File

@ -18,6 +18,9 @@ from test import inspect_fodder2 as mod2
# getclasstree, getargspec, getargvalues, formatargspec, formatargvalues,
# currentframe, stack, trace, isdatadescriptor
# NOTE: There are some additional tests relating to interaction with
# zipimport in the test_zipimport_support test module.
modfile = mod.__file__
if modfile.endswith(('c', 'o')):
modfile = modfile[:-1]

32
Lib/test/test_zipimport.py
View File

@ -211,16 +211,24 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
zi = zipimport.zipimporter(TEMP_ZIP)
self.assertEquals(zi.archive, TEMP_ZIP)
self.assertEquals(zi.is_package(TESTPACK), True)
zi.load_module(TESTPACK)
mod = zi.load_module(TESTPACK)
self.assertEquals(zi._get_filename(TESTPACK), mod.__file__)
self.assertEquals(zi.is_package(packdir + '__init__'), False)
self.assertEquals(zi.is_package(packdir + TESTPACK2), True)
self.assertEquals(zi.is_package(packdir2 + TESTMOD), False)
mod_name = packdir2 + TESTMOD
mod = __import__(module_path_to_dotted_name(mod_name))
mod_path = packdir2 + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
pkg = __import__(mod_name)
mod = sys.modules[mod_name]
self.assertEquals(zi.get_source(TESTPACK), None)
self.assertEquals(zi.get_source(mod_name), None)
self.assertEquals(zi.get_source(mod_path), None)
self.assertEquals(zi._get_filename(mod_path), mod.__file__)
# To pass in the module name instead of the path, we must use the right importer
loader = mod.__loader__
self.assertEquals(loader.get_source(mod_name), None)
self.assertEquals(loader._get_filename(mod_name), mod.__file__)
# test prefix and archivepath members
zi2 = zipimport.zipimporter(TEMP_ZIP + os.sep + TESTPACK)
@ -248,15 +256,23 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
self.assertEquals(zi.archive, TEMP_ZIP)
self.assertEquals(zi.prefix, packdir)
self.assertEquals(zi.is_package(TESTPACK2), True)
zi.load_module(TESTPACK2)
mod = zi.load_module(TESTPACK2)
self.assertEquals(zi._get_filename(TESTPACK2), mod.__file__)
self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
mod_name = TESTPACK2 + os.sep + TESTMOD
mod = __import__(module_path_to_dotted_name(mod_name))
mod_path = TESTPACK2 + os.sep + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
pkg = __import__(mod_name)
mod = sys.modules[mod_name]
self.assertEquals(zi.get_source(TESTPACK2), None)
self.assertEquals(zi.get_source(mod_name), None)
self.assertEquals(zi.get_source(mod_path), None)
self.assertEquals(zi._get_filename(mod_path), mod.__file__)
# To pass in the module name instead of the path, we must use the right importer
loader = mod.__loader__
self.assertEquals(loader.get_source(mod_name), None)
self.assertEquals(loader._get_filename(mod_name), mod.__file__)
finally:
z.close()
os.remove(TEMP_ZIP)

199
Lib/test/test_zipimport_support.py Normal file
View File

@ -0,0 +1,199 @@
# This test module covers support in various parts of the standard library
# for working with modules located inside zipfiles
# The tests are centralised in this fashion to make it easy to drop them
# if a platform doesn't support zipimport
import unittest
import test.support
import os
import os.path
import sys
import textwrap
import zipfile
import zipimport
import doctest
import inspect
import linecache
import pdb
verbose = test.support.verbose
# Library modules covered by this test set
# pdb (Issue 4201)
# inspect (Issue 4223)
# doctest (Issue 4197)
# Other test modules with zipimport related tests
# test_zipimport (of course!)
# test_cmd_line_script (covers the zipimport support in runpy)
# Retrieve some helpers from other test cases
from test import test_doctest, sample_doctest
from test.test_importhooks import ImportHooksBaseTestCase
from test.test_cmd_line_script import temp_dir, _run_python, \
_spawn_python, _kill_python, \
_make_test_script, \
_compile_test_script, \
_make_test_zip, _make_test_pkg
def _run_object_doctest(obj, module):
# Direct doctest output (normally just errors) to real stdout; doctest
# output shouldn't be compared by regrtest.
save_stdout = sys.stdout
sys.stdout = test.support.get_original_stdout()
try:
finder = doctest.DocTestFinder(verbose=verbose, recurse=False)
runner = doctest.DocTestRunner(verbose=verbose)
# Use the object's fully qualified name if it has one
# Otherwise, use the module's name
try:
name = "%s.%s" % (obj.__module__, obj.__name__)
except AttributeError:
name = module.__name__
for example in finder.find(obj, name, module):
runner.run(example)
f, t = runner.failures, runner.tries
if f:
raise test.support.TestFailed("%d of %d doctests failed" % (f, t))
finally:
sys.stdout = save_stdout
if verbose:
print ('doctest (%s) ... %d tests with zero failures' % (module.__name__, t))
return f, t
class ZipSupportTests(ImportHooksBaseTestCase):
# We use the ImportHooksBaseTestCase to restore
# the state of the import related information
# in the sys module after each test
# We also clear the linecache and zipimport cache
# just to avoid any bogus errors due to name reuse in the tests
def setUp(self):
linecache.clearcache()
zipimport._zip_directory_cache.clear()
ImportHooksBaseTestCase.setUp(self)
def test_inspect_getsource_issue4223(self):
test_src = "def foo(): pass\n"
with temp_dir() as d:
init_name = _make_test_script(d, '__init__', test_src)
name_in_zip = os.path.join('zip_pkg',
os.path.basename(init_name))
zip_name, run_name = _make_test_zip(d, 'test_zip',
init_name, name_in_zip)
os.remove(init_name)
sys.path.insert(0, zip_name)
import zip_pkg
self.assertEqual(inspect.getsource(zip_pkg.foo), test_src)
def test_doctest_issue4197(self):
# To avoid having to keep two copies of the doctest module's
# unit tests in sync, this test works by taking the source of
# test_doctest itself, rewriting it a bit to cope with a new
# location, and then throwing it in a zip file to make sure
# everything still works correctly
test_src = inspect.getsource(test_doctest)
test_src = test_src.replace(
"from test import test_doctest",
"import test_zipped_doctest as test_doctest")
test_src = test_src.replace("test.test_doctest",
"test_zipped_doctest")
test_src = test_src.replace("test.sample_doctest",
"sample_zipped_doctest")
sample_src = inspect.getsource(sample_doctest)
sample_src = sample_src.replace("test.test_doctest",
"test_zipped_doctest")
with temp_dir() as d:
script_name = _make_test_script(d, 'test_zipped_doctest',
test_src)
zip_name, run_name = _make_test_zip(d, 'test_zip',
script_name)
z = zipfile.ZipFile(zip_name, 'a')
z.writestr("sample_zipped_doctest.py", sample_src)
z.close()
if verbose:
zip_file = zipfile.ZipFile(zip_name, 'r')
print ('Contents of %r:' % zip_name)
zip_file.printdir()
zip_file.close()
os.remove(script_name)
sys.path.insert(0, zip_name)
import test_zipped_doctest
# Some of the doc tests depend on the colocated text files
# which aren't available to the zipped version (the doctest
# module currently requires real filenames for non-embedded
# tests). So we're forced to be selective about which tests
# to run.
# doctest could really use some APIs which take a text
# string or a file object instead of a filename...
known_good_tests = [
test_zipped_doctest.SampleClass,
test_zipped_doctest.SampleClass.NestedClass,
test_zipped_doctest.SampleClass.NestedClass.__init__,
test_zipped_doctest.SampleClass.__init__,
test_zipped_doctest.SampleClass.a_classmethod,
test_zipped_doctest.SampleClass.a_property,
test_zipped_doctest.SampleClass.a_staticmethod,
test_zipped_doctest.SampleClass.double,
test_zipped_doctest.SampleClass.get,
test_zipped_doctest.SampleNewStyleClass,
test_zipped_doctest.SampleNewStyleClass.__init__,
test_zipped_doctest.SampleNewStyleClass.double,
test_zipped_doctest.SampleNewStyleClass.get,
test_zipped_doctest.sample_func,
test_zipped_doctest.test_DocTest,
test_zipped_doctest.test_DocTestParser,
test_zipped_doctest.test_DocTestRunner.basics,
test_zipped_doctest.test_DocTestRunner.exceptions,
test_zipped_doctest.test_DocTestRunner.option_directives,
test_zipped_doctest.test_DocTestRunner.optionflags,
test_zipped_doctest.test_DocTestRunner.verbose_flag,
test_zipped_doctest.test_Example,
test_zipped_doctest.test_debug,
test_zipped_doctest.test_pdb_set_trace,
test_zipped_doctest.test_pdb_set_trace_nested,
test_zipped_doctest.test_testsource,
test_zipped_doctest.test_trailing_space_in_test,
test_zipped_doctest.test_DocTestSuite,
test_zipped_doctest.test_DocTestFinder,
]
# These remaining tests are the ones which need access
# to the data files, so we don't run them
fail_due_to_missing_data_files = [
test_zipped_doctest.test_DocFileSuite,
test_zipped_doctest.test_testfile,
test_zipped_doctest.test_unittest_reportflags,
]
for obj in known_good_tests:
_run_object_doctest(obj, test_zipped_doctest)
def test_pdb_issue4201(self):
test_src = textwrap.dedent("""\
def f():
pass
import pdb
pdb.runcall(f)
""")
with temp_dir() as d:
script_name = _make_test_script(d, 'script', test_src)
p = _spawn_python(script_name)
p.stdin.write(b'l\n')
data = _kill_python(p).decode()
self.assert_(script_name in data)
zip_name, run_name = _make_test_zip(d, "test_zip",
script_name, '__main__.py')
p = _spawn_python(zip_name)
p.stdin.write(b'l\n')
data = _kill_python(p).decode()
self.assert_(run_name in data)
def test_main():
test.support.run_unittest(ZipSupportTests)
test.support.reap_children()
if __name__ == '__main__':
test_main()

19
Misc/NEWS
View File

@ -45,6 +45,25 @@ Core and Builtins
Library
-------
- Issue #4223: inspect.getsource() will now correctly display source code
for packages loaded via zipimport (or any other conformant PEP 302
loader). Original patch by Alexander Belopolsky.
- Issue #4201: pdb can now access and display source code loaded via
zipimport (or any other conformant PEP 302 loader). Original patch by
Alexander Belopolsky.
- Issue #4197: doctests in modules loaded via zipimport (or any other PEP
302 conformant loader) will now work correctly in most cases (they
are still subject to the constraints that exist for all code running
from inside a module loaded via a PEP 302 loader and attempting to
perform IO operations based on __file__). Original patch by
Alexander Belopolsky.
- Issues #4082 and #4512: Add runpy support to zipimport in a manner that
allows backporting to maintenance branches. Original patch by
Alexander Belopolsky.
- Issue #4163: textwrap module: allow word splitting on a hyphen preceded by
a non-ASCII letter.

31
Modules/zipimport.c
View File

@ -354,6 +354,29 @@ error:
return NULL;
}
/* Return a string matching __file__ for the named module */
static PyObject *
zipimporter_get_filename(PyObject *obj, PyObject *args)
{
ZipImporter *self = (ZipImporter *)obj;
PyObject *code;
char *fullname, *modpath;
int ispackage;
if (!PyArg_ParseTuple(args, "s:zipimporter._get_filename",
&fullname))
return NULL;
/* Deciding the filename requires working out where the code
would come from if the module was actually loaded */
code = get_module_code(self, fullname, &ispackage, &modpath);
if (code == NULL)
return NULL;
Py_DECREF(code); /* Only need the path info */
return PyUnicode_FromString(modpath);
}
/* Return a bool signifying whether the module is a package or not. */
static PyObject *
zipimporter_is_package(PyObject *obj, PyObject *args)
@ -518,6 +541,12 @@ Return the source code for the specified module. Raise ZipImportError\n\
is the module couldn't be found, return None if the archive does\n\
contain the module, but has no source for it.");
PyDoc_STRVAR(doc_get_filename,
"_get_filename(fullname) -> filename string.\n\
\n\
Return the filename for the specified module.");
static PyMethodDef zipimporter_methods[] = {
{"find_module", zipimporter_find_module, METH_VARARGS,
doc_find_module},
@ -529,6 +558,8 @@ static PyMethodDef zipimporter_methods[] = {
doc_get_code},
{"get_source", zipimporter_get_source, METH_VARARGS,
doc_get_source},
{"_get_filename", zipimporter_get_filename, METH_VARARGS,
doc_get_filename},
{"is_package", zipimporter_is_package, METH_VARARGS,
doc_is_package},
{NULL, NULL} /* sentinel */