bpo-38112: Compileall improvements (GH-16012)
* Raise the limit of maximum path depth to actual recursion limit * Add posibilities to adjust a path compiled in .pyc file. Now, you can: - Strip a part of path from a beggining of path into compiled file example "-s /test /test/build/real/test.py" → "build/real/test.py" - Append some new path to a beggining of path into compiled file example "-p /boo real/test.py" → "/boo/real/test.py" You can also use both options in the same time. In that case, striping is done before appending. * Add a possibility to specify multiple optimization levels Each optimization level then leads to separated compiled file. Use `action='append'` instead of `nargs='+'` for the -o option. Instead of `-o 0 1 2`, specify `-o 0 -o 1 -o 2`. It's more to type, but much more explicit. * Add a symlinks limitation feature This feature allows us to limit byte-compilation of symbolic links if they are pointing outside specified dir (build root for example).
This commit is contained in:
Lumír 'Frenzy' Balhar
Petr Viktorin
parent
52b9408038
commit
8e7bb991de
|
|
@ -52,6 +52,13 @@ compile Python sources.
|
|||
cases where the source file does not exist at the time the byte-code file is
|
||||
executed.
|
||||
|
||||
.. cmdoption:: -s strip_prefix
|
||||
.. cmdoption:: -p prepend_prefix
|
||||
|
||||
Remove (``-s``) or append (``-p``) the given prefix of paths
|
||||
recorded in the ``.pyc`` files.
|
||||
Cannot be combined with ``-d``.
|
||||
|
||||
.. cmdoption:: -x regex
|
||||
|
||||
regex is used to search the full path to each file considered for
|
||||
|
|
@ -96,6 +103,16 @@ compile Python sources.
|
|||
variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH``
|
||||
environment variable is set.
|
||||
|
||||
.. cmdoption:: -o level
|
||||
|
||||
Compile with the given optimization level. May be used multiple times
|
||||
to compile for multiple levels at a time (for example,
|
||||
``compileall -o 1 -o 2``).
|
||||
|
||||
.. cmdoption:: -e dir
|
||||
|
||||
Ignore symlinks pointing outside the given directory.
|
||||
|
||||
.. versionchanged:: 3.2
|
||||
Added the ``-i``, ``-b`` and ``-h`` options.
|
||||
|
||||
|
|
@ -107,6 +124,12 @@ compile Python sources.
|
|||
.. versionchanged:: 3.7
|
||||
Added the ``--invalidation-mode`` option.
|
||||
|
||||
.. versionchanged:: 3.9
|
||||
Added the ``-s``, ``-p``, ``-e`` options.
|
||||
Raised the default recursion limit from 10 to
|
||||
:py:func:`sys.getrecursionlimit()`.
|
||||
Added the possibility to specify the ``-o`` option multiple times.
|
||||
|
||||
|
||||
There is no command-line option to control the optimization level used by the
|
||||
:func:`compile` function, because the Python interpreter itself already
|
||||
|
|
@ -120,7 +143,7 @@ runtime.
|
|||
Public functions
|
||||
----------------
|
||||
|
||||
.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None)
|
||||
.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, prependdir=None, limit_sl_dest=None)
|
||||
|
||||
Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
|
||||
files along the way. Return a true value if all the files compiled successfully,
|
||||
|
|
@ -166,6 +189,10 @@ Public functions
|
|||
:class:`py_compile.PycInvalidationMode` enum and controls how the generated
|
||||
pycs are invalidated at runtime.
|
||||
|
||||
The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to
|
||||
the ``-s``, ``-p`` and ``-e`` options described above.
|
||||
They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`.
|
||||
|
||||
.. versionchanged:: 3.2
|
||||
Added the *legacy* and *optimize* parameter.
|
||||
|
||||
|
|
@ -191,6 +218,9 @@ Public functions
|
|||
.. versionchanged:: 3.8
|
||||
Setting *workers* to 0 now chooses the optimal number of cores.
|
||||
|
||||
.. versionchanged:: 3.9
|
||||
Added *stripdir*, *prependdir* and *limit_sl_dest* arguments.
|
||||
|
||||
.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None)
|
||||
|
||||
Compile the file with path *fullname*. Return a true value if the file
|
||||
|
|
@ -223,6 +253,10 @@ Public functions
|
|||
:class:`py_compile.PycInvalidationMode` enum and controls how the generated
|
||||
pycs are invalidated at runtime.
|
||||
|
||||
The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to
|
||||
the ``-s``, ``-p`` and ``-e`` options described above.
|
||||
They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`.
|
||||
|
||||
.. versionadded:: 3.2
|
||||
|
||||
.. versionchanged:: 3.5
|
||||
|
|
@ -238,6 +272,9 @@ Public functions
|
|||
.. versionchanged:: 3.7.2
|
||||
The *invalidation_mode* parameter's default value is updated to None.
|
||||
|
||||
.. versionchanged:: 3.9
|
||||
Added *stripdir*, *prependdir* and *limit_sl_dest* arguments.
|
||||
|
||||
.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None)
|
||||
|
||||
Byte-compile all the :file:`.py` files found along ``sys.path``. Return a
|
||||
|
|
|
|||
|
|
@ -17,10 +17,13 @@ import py_compile
|
|||
import struct
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
RECURSION_LIMIT = sys.getrecursionlimit()
|
||||
|
||||
__all__ = ["compile_dir","compile_file","compile_path"]
|
||||
|
||||
def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
|
||||
def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
|
||||
if quiet < 2 and isinstance(dir, os.PathLike):
|
||||
dir = os.fspath(dir)
|
||||
if not quiet:
|
||||
|
|
@ -36,35 +39,39 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
|
|||
if name == '__pycache__':
|
||||
continue
|
||||
fullname = os.path.join(dir, name)
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
else:
|
||||
dfile = None
|
||||
if not os.path.isdir(fullname):
|
||||
yield fullname
|
||||
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
|
||||
os.path.isdir(fullname) and not os.path.islink(fullname)):
|
||||
yield from _walk_dir(fullname, ddir=dfile,
|
||||
maxlevels=maxlevels - 1, quiet=quiet)
|
||||
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
|
||||
quiet=quiet)
|
||||
|
||||
def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
|
||||
quiet=0, legacy=False, optimize=-1, workers=1,
|
||||
invalidation_mode=None):
|
||||
def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
|
||||
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
|
||||
invalidation_mode=None, stripdir=None,
|
||||
prependdir=None, limit_sl_dest=None):
|
||||
"""Byte-compile all modules in the given directory tree.
|
||||
|
||||
Arguments (only dir is required):
|
||||
|
||||
dir: the directory to byte-compile
|
||||
maxlevels: maximum recursion level (default 10)
|
||||
maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
|
||||
ddir: the directory that will be prepended to the path to the
|
||||
file as it is compiled into each byte-code file.
|
||||
force: if True, force compilation, even if timestamps are up-to-date
|
||||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
workers: maximum number of parallel workers
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beggining of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path
|
||||
"""
|
||||
ProcessPoolExecutor = None
|
||||
if workers < 0:
|
||||
|
|
@ -76,8 +83,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
|
|||
from concurrent.futures import ProcessPoolExecutor
|
||||
except ImportError:
|
||||
workers = 1
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
|
||||
ddir=ddir)
|
||||
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
|
||||
success = True
|
||||
if workers != 1 and ProcessPoolExecutor is not None:
|
||||
# If workers == 0, let ProcessPoolExecutor choose
|
||||
|
|
@ -88,19 +94,25 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
|
|||
rx=rx, quiet=quiet,
|
||||
legacy=legacy,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode),
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=stripdir,
|
||||
prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest),
|
||||
files)
|
||||
success = min(results, default=True)
|
||||
else:
|
||||
for file in files:
|
||||
if not compile_file(file, ddir, force, rx, quiet,
|
||||
legacy, optimize, invalidation_mode):
|
||||
legacy, optimize, invalidation_mode,
|
||||
stripdir=stripdir, prependdir=prependdir,
|
||||
limit_sl_dest=limit_sl_dest):
|
||||
success = False
|
||||
return success
|
||||
|
||||
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
||||
legacy=False, optimize=-1,
|
||||
invalidation_mode=None):
|
||||
invalidation_mode=None, stripdir=None, prependdir=None,
|
||||
limit_sl_dest=None):
|
||||
"""Byte-compile one file.
|
||||
|
||||
Arguments (only fullname is required):
|
||||
|
|
@ -112,32 +124,76 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
quiet: full output with False or 0, errors only with 1,
|
||||
no output with 2
|
||||
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
|
||||
optimize: optimization level or -1 for level of the interpreter
|
||||
optimize: int or list of optimization levels or -1 for level of
|
||||
the interpreter. Multiple levels leads to multiple compiled
|
||||
files each with one optimization level.
|
||||
invalidation_mode: how the up-to-dateness of the pyc will be checked
|
||||
stripdir: part of path to left-strip from source file path
|
||||
prependdir: path to prepend to beggining of original file path, applied
|
||||
after stripdir
|
||||
limit_sl_dest: ignore symlinks if they are pointing outside of
|
||||
the defined path.
|
||||
"""
|
||||
|
||||
if ddir is not None and (stripdir is not None or prependdir is not None):
|
||||
raise ValueError(("Destination dir (ddir) cannot be used "
|
||||
"in combination with stripdir or prependdir"))
|
||||
|
||||
success = True
|
||||
if quiet < 2 and isinstance(fullname, os.PathLike):
|
||||
fullname = os.fspath(fullname)
|
||||
name = os.path.basename(fullname)
|
||||
|
||||
dfile = None
|
||||
|
||||
if ddir is not None:
|
||||
dfile = os.path.join(ddir, name)
|
||||
else:
|
||||
dfile = None
|
||||
|
||||
if stripdir is not None:
|
||||
fullname_parts = fullname.split(os.path.sep)
|
||||
stripdir_parts = stripdir.split(os.path.sep)
|
||||
ddir_parts = list(fullname_parts)
|
||||
|
||||
for spart, opart in zip(stripdir_parts, fullname_parts):
|
||||
if spart == opart:
|
||||
ddir_parts.remove(spart)
|
||||
|
||||
dfile = os.path.join(*ddir_parts)
|
||||
|
||||
if prependdir is not None:
|
||||
if dfile is None:
|
||||
dfile = os.path.join(prependdir, fullname)
|
||||
else:
|
||||
dfile = os.path.join(prependdir, dfile)
|
||||
|
||||
if isinstance(optimize, int):
|
||||
optimize = [optimize]
|
||||
|
||||
if rx is not None:
|
||||
mo = rx.search(fullname)
|
||||
if mo:
|
||||
return success
|
||||
|
||||
if limit_sl_dest is not None and os.path.islink(fullname):
|
||||
if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
|
||||
return success
|
||||
|
||||
opt_cfiles = {}
|
||||
|
||||
if os.path.isfile(fullname):
|
||||
if legacy:
|
||||
cfile = fullname + 'c'
|
||||
else:
|
||||
if optimize >= 0:
|
||||
opt = optimize if optimize >= 1 else ''
|
||||
cfile = importlib.util.cache_from_source(
|
||||
fullname, optimization=opt)
|
||||
for opt_level in optimize:
|
||||
if legacy:
|
||||
opt_cfiles[opt_level] = fullname + 'c'
|
||||
else:
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
cache_dir = os.path.dirname(cfile)
|
||||
if opt_level >= 0:
|
||||
opt = opt_level if opt_level >= 1 else ''
|
||||
cfile = (importlib.util.cache_from_source(
|
||||
fullname, optimization=opt))
|
||||
opt_cfiles[opt_level] = cfile
|
||||
else:
|
||||
cfile = importlib.util.cache_from_source(fullname)
|
||||
opt_cfiles[opt_level] = cfile
|
||||
|
||||
head, tail = name[:-3], name[-3:]
|
||||
if tail == '.py':
|
||||
if not force:
|
||||
|
|
@ -145,18 +201,22 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
|
|||
mtime = int(os.stat(fullname).st_mtime)
|
||||
expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER,
|
||||
0, mtime)
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(12)
|
||||
if expect == actual:
|
||||
for cfile in opt_cfiles.values():
|
||||
with open(cfile, 'rb') as chandle:
|
||||
actual = chandle.read(12)
|
||||
if expect != actual:
|
||||
break
|
||||
else:
|
||||
return success
|
||||
except OSError:
|
||||
pass
|
||||
if not quiet:
|
||||
print('Compiling {!r}...'.format(fullname))
|
||||
try:
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=optimize,
|
||||
invalidation_mode=invalidation_mode)
|
||||
for opt_level, cfile in opt_cfiles.items():
|
||||
ok = py_compile.compile(fullname, cfile, dfile, True,
|
||||
optimize=opt_level,
|
||||
invalidation_mode=invalidation_mode)
|
||||
except py_compile.PyCompileError as err:
|
||||
success = False
|
||||
if quiet >= 2:
|
||||
|
|
@ -225,7 +285,7 @@ def main():
|
|||
parser = argparse.ArgumentParser(
|
||||
description='Utilities to support installing Python libraries.')
|
||||
parser.add_argument('-l', action='store_const', const=0,
|
||||
default=10, dest='maxlevels',
|
||||
default=RECURSION_LIMIT, dest='maxlevels',
|
||||
help="don't recurse into subdirectories")
|
||||
parser.add_argument('-r', type=int, dest='recursion',
|
||||
help=('control the maximum recursion level. '
|
||||
|
|
@ -243,6 +303,20 @@ def main():
|
|||
'compile-time tracebacks and in runtime '
|
||||
'tracebacks in cases where the source file is '
|
||||
'unavailable'))
|
||||
parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',
|
||||
default=None,
|
||||
help=('part of path to left-strip from path '
|
||||
'to source file - for example buildroot. '
|
||||
'`-d` and `-s` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
|
||||
default=None,
|
||||
help=('path to add as prefix to path '
|
||||
'to source file - for example / to make '
|
||||
'it absolute when some part is removed '
|
||||
'by `-s` option. '
|
||||
'`-d` and `-p` options cannot be '
|
||||
'specified together.'))
|
||||
parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
|
||||
help=('skip files matching the regular expression; '
|
||||
'the regexp is searched for in the full path '
|
||||
|
|
@ -265,6 +339,12 @@ def main():
|
|||
'"checked-hash" if the SOURCE_DATE_EPOCH '
|
||||
'environment variable is set, and '
|
||||
'"timestamp" otherwise.'))
|
||||
parser.add_argument('-o', action='append', type=int, dest='opt_levels',
|
||||
help=('Optimization levels to run compilation with.'
|
||||
'Default is -1 which uses optimization level of'
|
||||
'Python interpreter itself (specified by -O).'))
|
||||
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
|
||||
help='Ignore symlinks pointing outsite of the DIR')
|
||||
|
||||
args = parser.parse_args()
|
||||
compile_dests = args.compile_dest
|
||||
|
|
@ -273,12 +353,22 @@ def main():
|
|||
import re
|
||||
args.rx = re.compile(args.rx)
|
||||
|
||||
if args.limit_sl_dest == "":
|
||||
args.limit_sl_dest = None
|
||||
|
||||
if args.recursion is not None:
|
||||
maxlevels = args.recursion
|
||||
else:
|
||||
maxlevels = args.maxlevels
|
||||
|
||||
if args.opt_levels is None:
|
||||
args.opt_levels = [-1]
|
||||
|
||||
if args.ddir is not None and (
|
||||
args.stripdir is not None or args.prependdir is not None
|
||||
):
|
||||
parser.error("-d cannot be used in combination with -s or -p")
|
||||
|
||||
# if flist is provided then load it
|
||||
if args.flist:
|
||||
try:
|
||||
|
|
@ -303,13 +393,21 @@ def main():
|
|||
if os.path.isfile(dest):
|
||||
if not compile_file(dest, args.ddir, args.force, args.rx,
|
||||
args.quiet, args.legacy,
|
||||
invalidation_mode=invalidation_mode):
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest):
|
||||
success = False
|
||||
else:
|
||||
if not compile_dir(dest, maxlevels, args.ddir,
|
||||
args.force, args.rx, args.quiet,
|
||||
args.legacy, workers=args.workers,
|
||||
invalidation_mode=invalidation_mode):
|
||||
invalidation_mode=invalidation_mode,
|
||||
stripdir=args.stripdir,
|
||||
prependdir=args.prependdir,
|
||||
optimize=args.opt_levels,
|
||||
limit_sl_dest=args.limit_sl_dest):
|
||||
success = False
|
||||
return success
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,16 @@ class CompileallTestsBase:
|
|||
os.mkdir(self.subdirectory)
|
||||
self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
|
||||
shutil.copyfile(self.source_path, self.source_path3)
|
||||
many_directories = [str(number) for number in range(1, 100)]
|
||||
self.long_path = os.path.join(self.directory,
|
||||
"long",
|
||||
*many_directories)
|
||||
os.makedirs(self.long_path)
|
||||
self.source_path_long = os.path.join(self.long_path, '_test4.py')
|
||||
shutil.copyfile(self.source_path, self.source_path_long)
|
||||
self.bc_path_long = importlib.util.cache_from_source(
|
||||
self.source_path_long
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.directory)
|
||||
|
|
@ -194,6 +204,123 @@ class CompileallTestsBase:
|
|||
compileall.compile_dir(self.directory, quiet=True, workers=5)
|
||||
self.assertTrue(compile_file_mock.called)
|
||||
|
||||
def text_compile_dir_maxlevels(self):
|
||||
# Test the actual impact of maxlevels attr
|
||||
compileall.compile_dir(os.path.join(self.directory, "long"),
|
||||
maxlevels=10, quiet=True)
|
||||
self.assertFalse(os.path.isfile(self.bc_path_long))
|
||||
compileall.compile_dir(os.path.join(self.directory, "long"),
|
||||
maxlevels=110, quiet=True)
|
||||
self.assertTrue(os.path.isfile(self.bc_path_long))
|
||||
|
||||
def test_strip_only(self):
|
||||
fullpath = ["test", "build", "real", "path"]
|
||||
path = os.path.join(self.directory, *fullpath)
|
||||
os.makedirs(path)
|
||||
script = script_helper.make_script(path, "test", "1 / 0")
|
||||
bc = importlib.util.cache_from_source(script)
|
||||
stripdir = os.path.join(self.directory, *fullpath[:2])
|
||||
compileall.compile_dir(path, quiet=True, stripdir=stripdir)
|
||||
rc, out, err = script_helper.assert_python_failure(bc)
|
||||
expected_in = os.path.join(*fullpath[2:])
|
||||
self.assertIn(
|
||||
expected_in,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
self.assertNotIn(
|
||||
stripdir,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
|
||||
def test_prepend_only(self):
|
||||
fullpath = ["test", "build", "real", "path"]
|
||||
path = os.path.join(self.directory, *fullpath)
|
||||
os.makedirs(path)
|
||||
script = script_helper.make_script(path, "test", "1 / 0")
|
||||
bc = importlib.util.cache_from_source(script)
|
||||
prependdir = "/foo"
|
||||
compileall.compile_dir(path, quiet=True, prependdir=prependdir)
|
||||
rc, out, err = script_helper.assert_python_failure(bc)
|
||||
expected_in = os.path.join(prependdir, self.directory, *fullpath)
|
||||
self.assertIn(
|
||||
expected_in,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
|
||||
def test_strip_and_prepend(self):
|
||||
fullpath = ["test", "build", "real", "path"]
|
||||
path = os.path.join(self.directory, *fullpath)
|
||||
os.makedirs(path)
|
||||
script = script_helper.make_script(path, "test", "1 / 0")
|
||||
bc = importlib.util.cache_from_source(script)
|
||||
stripdir = os.path.join(self.directory, *fullpath[:2])
|
||||
prependdir = "/foo"
|
||||
compileall.compile_dir(path, quiet=True,
|
||||
stripdir=stripdir, prependdir=prependdir)
|
||||
rc, out, err = script_helper.assert_python_failure(bc)
|
||||
expected_in = os.path.join(prependdir, *fullpath[2:])
|
||||
self.assertIn(
|
||||
expected_in,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
self.assertNotIn(
|
||||
stripdir,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
|
||||
def test_strip_prepend_and_ddir(self):
|
||||
fullpath = ["test", "build", "real", "path", "ddir"]
|
||||
path = os.path.join(self.directory, *fullpath)
|
||||
os.makedirs(path)
|
||||
script_helper.make_script(path, "test", "1 / 0")
|
||||
with self.assertRaises(ValueError):
|
||||
compileall.compile_dir(path, quiet=True, ddir="/bar",
|
||||
stripdir="/foo", prependdir="/bar")
|
||||
|
||||
def test_multiple_optimization_levels(self):
|
||||
script = script_helper.make_script(self.directory,
|
||||
"test_optimization",
|
||||
"a = 0")
|
||||
bc = []
|
||||
for opt_level in "", 1, 2, 3:
|
||||
bc.append(importlib.util.cache_from_source(script,
|
||||
optimization=opt_level))
|
||||
test_combinations = [[0, 1], [1, 2], [0, 2], [0, 1, 2]]
|
||||
for opt_combination in test_combinations:
|
||||
compileall.compile_file(script, quiet=True,
|
||||
optimize=opt_combination)
|
||||
for opt_level in opt_combination:
|
||||
self.assertTrue(os.path.isfile(bc[opt_level]))
|
||||
try:
|
||||
os.unlink(bc[opt_level])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@support.skip_unless_symlink
|
||||
def test_ignore_symlink_destination(self):
|
||||
# Create folders for allowed files, symlinks and prohibited area
|
||||
allowed_path = os.path.join(self.directory, "test", "dir", "allowed")
|
||||
symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks")
|
||||
prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited")
|
||||
os.makedirs(allowed_path)
|
||||
os.makedirs(symlinks_path)
|
||||
os.makedirs(prohibited_path)
|
||||
|
||||
# Create scripts and symlinks and remember their byte-compiled versions
|
||||
allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0")
|
||||
prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0")
|
||||
allowed_symlink = os.path.join(symlinks_path, "test_allowed.py")
|
||||
prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py")
|
||||
os.symlink(allowed_script, allowed_symlink)
|
||||
os.symlink(prohibited_script, prohibited_symlink)
|
||||
allowed_bc = importlib.util.cache_from_source(allowed_symlink)
|
||||
prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)
|
||||
|
||||
compileall.compile_dir(symlinks_path, quiet=True, limit_sl_dest=allowed_path)
|
||||
|
||||
self.assertTrue(os.path.isfile(allowed_bc))
|
||||
self.assertFalse(os.path.isfile(prohibited_bc))
|
||||
|
||||
|
||||
class CompileallTestsWithSourceEpoch(CompileallTestsBase,
|
||||
unittest.TestCase,
|
||||
|
|
@ -436,6 +563,20 @@ class CommandLineTestsBase:
|
|||
self.assertCompiled(spamfn)
|
||||
self.assertCompiled(eggfn)
|
||||
|
||||
@support.skip_unless_symlink
|
||||
def test_symlink_loop(self):
|
||||
# Currently, compileall ignores symlinks to directories.
|
||||
# If that limitation is ever lifted, it should protect against
|
||||
# recursion in symlink loops.
|
||||
pkg = os.path.join(self.pkgdir, 'spam')
|
||||
script_helper.make_pkg(pkg)
|
||||
os.symlink('.', os.path.join(pkg, 'evil'))
|
||||
os.symlink('.', os.path.join(pkg, 'evil2'))
|
||||
self.assertRunOK('-q', self.pkgdir)
|
||||
self.assertCompiled(os.path.join(
|
||||
self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
|
||||
))
|
||||
|
||||
def test_quiet(self):
|
||||
noisy = self.assertRunOK(self.pkgdir)
|
||||
quiet = self.assertRunOK('-q', self.pkgdir)
|
||||
|
|
@ -577,6 +718,74 @@ class CommandLineTestsBase:
|
|||
self.assertTrue(compile_dir.called)
|
||||
self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
|
||||
|
||||
def test_strip_and_prepend(self):
|
||||
fullpath = ["test", "build", "real", "path"]
|
||||
path = os.path.join(self.directory, *fullpath)
|
||||
os.makedirs(path)
|
||||
script = script_helper.make_script(path, "test", "1 / 0")
|
||||
bc = importlib.util.cache_from_source(script)
|
||||
stripdir = os.path.join(self.directory, *fullpath[:2])
|
||||
prependdir = "/foo"
|
||||
self.assertRunOK("-s", stripdir, "-p", prependdir, path)
|
||||
rc, out, err = script_helper.assert_python_failure(bc)
|
||||
expected_in = os.path.join(prependdir, *fullpath[2:])
|
||||
self.assertIn(
|
||||
expected_in,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
self.assertNotIn(
|
||||
stripdir,
|
||||
str(err, encoding=sys.getdefaultencoding())
|
||||
)
|
||||
|
||||
def test_multiple_optimization_levels(self):
|
||||
path = os.path.join(self.directory, "optimizations")
|
||||
os.makedirs(path)
|
||||
script = script_helper.make_script(path,
|
||||
"test_optimization",
|
||||
"a = 0")
|
||||
bc = []
|
||||
for opt_level in "", 1, 2, 3:
|
||||
bc.append(importlib.util.cache_from_source(script,
|
||||
optimization=opt_level))
|
||||
test_combinations = [["0", "1"],
|
||||
["1", "2"],
|
||||
["0", "2"],
|
||||
["0", "1", "2"]]
|
||||
for opt_combination in test_combinations:
|
||||
self.assertRunOK(path, *("-o" + str(n) for n in opt_combination))
|
||||
for opt_level in opt_combination:
|
||||
self.assertTrue(os.path.isfile(bc[int(opt_level)]))
|
||||
try:
|
||||
os.unlink(bc[opt_level])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@support.skip_unless_symlink
|
||||
def test_ignore_symlink_destination(self):
|
||||
# Create folders for allowed files, symlinks and prohibited area
|
||||
allowed_path = os.path.join(self.directory, "test", "dir", "allowed")
|
||||
symlinks_path = os.path.join(self.directory, "test", "dir", "symlinks")
|
||||
prohibited_path = os.path.join(self.directory, "test", "dir", "prohibited")
|
||||
os.makedirs(allowed_path)
|
||||
os.makedirs(symlinks_path)
|
||||
os.makedirs(prohibited_path)
|
||||
|
||||
# Create scripts and symlinks and remember their byte-compiled versions
|
||||
allowed_script = script_helper.make_script(allowed_path, "test_allowed", "a = 0")
|
||||
prohibited_script = script_helper.make_script(prohibited_path, "test_prohibited", "a = 0")
|
||||
allowed_symlink = os.path.join(symlinks_path, "test_allowed.py")
|
||||
prohibited_symlink = os.path.join(symlinks_path, "test_prohibited.py")
|
||||
os.symlink(allowed_script, allowed_symlink)
|
||||
os.symlink(prohibited_script, prohibited_symlink)
|
||||
allowed_bc = importlib.util.cache_from_source(allowed_symlink)
|
||||
prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)
|
||||
|
||||
self.assertRunOK(symlinks_path, "-e", allowed_path)
|
||||
|
||||
self.assertTrue(os.path.isfile(allowed_bc))
|
||||
self.assertFalse(os.path.isfile(prohibited_bc))
|
||||
|
||||
|
||||
class CommandLineTestsWithSourceEpoch(CommandLineTestsBase,
|
||||
unittest.TestCase,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
:mod:`compileall` has a higher default recursion limit and new command-line
|
||||
arguments for path manipulation, symlinks handling, and multiple
|
||||
optimization levels.
|
||||
Loading…
Reference in New Issue