From 8334fd9285a8e9f0864b0453ae738fe3f6893b21 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 4 Dec 2010 10:26:46 +0000 Subject: [PATCH] Add an "optimize" parameter to compile() to control the optimization level, and provide an interface to it in py_compile, compileall and PyZipFile. --- Doc/c-api/veryhigh.rst | 14 ++++++ Doc/library/compileall.rst | 19 +++++--- Doc/library/functions.rst | 10 ++++- Doc/library/py_compile.rst | 9 +++- Doc/library/zipfile.rst | 59 +++++++++++++++--------- Include/compile.h | 5 ++- Include/pythonrun.h | 7 +-- Lib/compileall.py | 25 +++++++---- Lib/py_compile.py | 14 ++++-- Lib/test/test_builtin.py | 29 ++++++++++++ Lib/test/test_compileall.py | 9 ++++ Lib/test/test_zipfile.py | 16 +++++++ Lib/zipfile.py | 89 ++++++++++++++++++++++++------------- Misc/NEWS | 5 +++ Python/bltinmodule.c | 19 +++++--- Python/compile.c | 32 ++++++++----- Python/pythonrun.c | 16 +++++-- 17 files changed, 280 insertions(+), 97 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 5b9332560ce..3e41ec73f07 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -230,6 +230,12 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) + This is a simplified interface to :c:func:`Py_CompileStringExFlags` below, with + *optimize* set to ``-1``. + + +.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) + Parse and compile the Python source code in *str*, returning the resulting code object. The start token is given by *start*; this can be used to constrain the code which can be compiled and should be :const:`Py_eval_input`, @@ -238,6 +244,14 @@ the same library that the Python runtime is using. :exc:`SyntaxError` exception messages. This returns *NULL* if the code cannot be parsed or compiled. + The integer *optimize* specifies the optimization level of the compiler; a + value of ``-1`` selects the optimization level of the interpreter as given by + :option:`-O` options. Explicit levels are ``0`` (no optimization; + ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) + or ``2`` (docstrings are removed too). + + .. versionadded:: 3.2 + .. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 63ddc99cdd1..1835b315a96 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -58,7 +58,7 @@ compile Python sources. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. The *maxlevels* parameter is used to limit the depth of @@ -76,14 +76,23 @@ Public functions If *legacy* is true, old-style ``.pyc`` file path names are written, otherwise (the default), :pep:`3147`-style path names are written. + *optimize* specifies the optimization level for the compiler. It is passed to + the built-in :func:`compile` function. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False) + .. versionchanged:: 3.2 + Added the *optimize* parameter. + + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False, optimize=-1) Byte-compile all the :file:`.py` files found along ``sys.path``. If *skip_curdir* is true (the default), the current directory is not included in - the search. The *maxlevels* parameter defaults to ``0``, and the *force* - and *legacy* parameters default to ``False``. All are - passed to the :func:`compile_dir` function. + the search. All other parameters are passed to the :func:`compile_dir` + function. + + .. versionchanged:: 3.2 + Added the *optimize* parameter. + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7579cc95577..1303f1c40f2 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -174,7 +174,7 @@ are always available. They are listed here in alphabetical order. type hierarchy in :ref:`types`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False) +.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a string or an AST @@ -206,6 +206,12 @@ are always available. They are listed here in alphabetical order. can be found as the :attr:`compiler_flag` attribute on the :class:`_Feature` instance in the :mod:`__future__` module. + The argument *optimize* specifies the optimization level of the compiler; the + default value of ``-1`` selects the optimization level of the interpreter as + given by :option:`-O` options. Explicit levels are ``0`` (no optimization; + ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) + or ``2`` (docstrings are removed too). + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`TypeError` if the source contains null bytes. @@ -218,7 +224,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode - does not have to end in a newline anymore. + does not have to end in a newline anymore. Added the *optimize* parameter. .. function:: complex([real[, imag]]) diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index c6eea8499a1..b5b90107eca 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -22,7 +22,7 @@ byte-code cache files in the directory containing the source code. Exception raised when an error occurs while attempting to compile the file. -.. function:: compile(file, cfile=None, dfile=None, doraise=False) +.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file name *file*. The byte-code is written to @@ -37,6 +37,13 @@ byte-code cache files in the directory containing the source code. returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + *optimize* controls the optimization level and is passed to the built-in + :func:`compile` function. The default of ``-1`` selects the optimization + level of the current interpreter. + + .. versionchanged:: 3.2 + Added the *optimize* parameter. + .. function:: main(args=None) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index bb557d4ecdb..109bef05957 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -51,6 +51,7 @@ The module defines the following items: .. class:: PyZipFile + :noindex: Class for creating ZIP archives containing Python libraries. @@ -318,37 +319,53 @@ The following data attributes are also available: string no longer than 65535 bytes. Comments longer than this will be truncated in the written archive when :meth:`ZipFile.close` is called. + .. _pyzipfile-objects: PyZipFile Objects ----------------- The :class:`PyZipFile` constructor takes the same parameters as the -:class:`ZipFile` constructor. Instances have one method in addition to those of -:class:`ZipFile` objects. +:class:`ZipFile` constructor, and one additional parameter, *optimize*. +.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False, \ + optimize=-1) -.. method:: PyZipFile.writepy(pathname, basename='') + .. versionadded:: 3.2 + The *optimize* parameter. - Search for files :file:`\*.py` and add the corresponding file to the archive. - The corresponding file is a :file:`\*.pyo` file if available, else a - :file:`\*.pyc` file, compiling if necessary. If the pathname is a file, the - filename must end with :file:`.py`, and just the (corresponding - :file:`\*.py[co]`) file is added at the top level (no path information). If the - pathname is a file that does not end with :file:`.py`, a :exc:`RuntimeError` - will be raised. If it is a directory, and the directory is not a package - directory, then all the files :file:`\*.py[co]` are added at the top level. If - the directory is a package directory, then all :file:`\*.py[co]` are added under - the package name as a file path, and if any subdirectories are package - directories, all of these are added recursively. *basename* is intended for - internal use only. The :meth:`writepy` method makes archives with file names - like this:: + Instances have one method in addition to those of :class:`ZipFile` objects: - string.pyc # Top level name - test/__init__.pyc # Package directory - test/testall.pyc # Module test.testall - test/bogus/__init__.pyc # Subpackage directory - test/bogus/myfile.pyc # Submodule test.bogus.myfile + .. method:: PyZipFile.writepy(pathname, basename='') + + Search for files :file:`\*.py` and add the corresponding file to the + archive. + + If the *optimize* parameter to :class:`PyZipFile` was not given or ``-1``, + the corresponding file is a :file:`\*.pyo` file if available, else a + :file:`\*.pyc` file, compiling if necessary. + + If the *optimize* parameter to :class:`PyZipFile` was ``0``, ``1`` or + ``2``, only files with that optimization level (see :func:`compile`) are + added to the archive, compiling if necessary. + + If the pathname is a file, the filename must end with :file:`.py`, and + just the (corresponding :file:`\*.py[co]`) file is added at the top level + (no path information). If the pathname is a file that does not end with + :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, + and the directory is not a package directory, then all the files + :file:`\*.py[co]` are added at the top level. If the directory is a + package directory, then all :file:`\*.py[co]` are added under the package + name as a file path, and if any subdirectories are package directories, + all of these are added recursively. *basename* is intended for internal + use only. The :meth:`writepy` method makes archives with file names like + this:: + + string.pyc # Top level name + test/__init__.pyc # Package directory + test/testall.pyc # Module test.testall + test/bogus/__init__.pyc # Subpackage directory + test/bogus/myfile.pyc # Submodule test.bogus.myfile .. _zipinfo-objects: diff --git a/Include/compile.h b/Include/compile.h index 4a5ebe67783..456a4940284 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -29,8 +29,9 @@ typedef struct { #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" struct _mod; /* Declare the existence of this type */ -PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *, - PyCompilerFlags *, PyArena *); +#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) +PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *, + PyCompilerFlags *, int, PyArena *); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 95bdf9e219e..4eae54922e5 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -76,9 +76,10 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, #ifdef Py_LIMITED_API PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int); #else -#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL) -PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int, - PyCompilerFlags *); +#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) +#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) +PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int, + PyCompilerFlags *, int); #endif PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int); diff --git a/Lib/compileall.py b/Lib/compileall.py index 17cc61dc1a0..aefdb89706d 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -19,8 +19,8 @@ import struct __all__ = ["compile_dir","compile_file","compile_path"] -def compile_dir(dir, maxlevels=10, ddir=None, - force=False, rx=None, quiet=False, legacy=False): +def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, + quiet=False, legacy=False, optimize=-1): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -32,6 +32,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force: if True, force compilation, even if timestamps are up-to-date quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: optimization level or -1 for level of the interpreter """ if not quiet: print('Listing', dir, '...') @@ -51,7 +52,8 @@ def compile_dir(dir, maxlevels=10, ddir=None, else: dfile = None if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, legacy): + if not compile_file(fullname, ddir, force, rx, quiet, + legacy, optimize): success = 0 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): @@ -61,7 +63,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, return success def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, - legacy=False): + legacy=False, optimize=-1): """Byte-compile file. fullname: the file to byte-compile ddir: if given, purported directory name (this is the @@ -69,6 +71,7 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, force: if True, force compilation, even if timestamps are up-to-date quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: optimization level or -1 for level of the interpreter """ success = 1 name = os.path.basename(fullname) @@ -84,7 +87,11 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, if legacy: cfile = fullname + ('c' if __debug__ else 'o') else: - cfile = imp.cache_from_source(fullname) + if optimize >= 0: + cfile = imp.cache_from_source(fullname, + debug_override=not optimize) + else: + cfile = imp.cache_from_source(fullname) cache_dir = os.path.dirname(cfile) head, tail = name[:-3], name[-3:] if tail == '.py': @@ -101,7 +108,8 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, if not quiet: print('Compiling', fullname, '...') try: - ok = py_compile.compile(fullname, cfile, dfile, True) + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=optimize) except py_compile.PyCompileError as err: if quiet: print('*** Error compiling', fullname, '...') @@ -126,7 +134,7 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, - legacy=False): + legacy=False, optimize=-1): """Byte-compile all module on sys.path. Arguments (all optional): @@ -136,6 +144,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, force: as for compile_dir() (default False) quiet: as for compile_dir() (default False) legacy: as for compile_dir() (default False) + optimize: as for compile_dir() (default -1) """ success = 1 for dir in sys.path: @@ -144,7 +153,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, else: success = success and compile_dir(dir, maxlevels, None, force, quiet=quiet, - legacy=legacy) + legacy=legacy, optimize=optimize) return success diff --git a/Lib/py_compile.py b/Lib/py_compile.py index d241434a602..e0f98cb741d 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -72,7 +72,7 @@ def wr_long(f, x): (x >> 16) & 0xff, (x >> 24) & 0xff])) -def compile(file, cfile=None, dfile=None, doraise=False): +def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): """Byte-compile one Python source file to Python bytecode. :param file: The source file name. @@ -86,6 +86,10 @@ def compile(file, cfile=None, dfile=None, doraise=False): will be printed, and the function will return to the caller. If an exception occurs and this flag is set to True, a PyCompileError exception will be raised. + :param optimize: The optimization level for the compiler. Valid values + are -1, 0, 1 and 2. A value of -1 means to use the optimization + level of the current interpreter, as given by -O command line options. + :return: Path to the resulting byte compiled file. Note that it isn't necessary to byte-compile Python modules for @@ -111,7 +115,8 @@ def compile(file, cfile=None, dfile=None, doraise=False): timestamp = int(os.stat(file).st_mtime) codestring = f.read() try: - codeobject = builtins.compile(codestring, dfile or file,'exec') + codeobject = builtins.compile(codestring, dfile or file, 'exec', + optimize=optimize) except Exception as err: py_exc = PyCompileError(err.__class__, err, dfile or file) if doraise: @@ -120,7 +125,10 @@ def compile(file, cfile=None, dfile=None, doraise=False): sys.stderr.write(py_exc.msg + '\n') return if cfile is None: - cfile = imp.cache_from_source(file) + if optimize >= 0: + cfile = imp.cache_from_source(file, debug_override=not optimize) + else: + cfile = imp.cache_from_source(file) try: os.makedirs(os.path.dirname(cfile)) except OSError as error: diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 7b73949267c..1469e36847a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -6,6 +6,7 @@ import sys import warnings import collections import io +import ast import types import builtins import random @@ -285,6 +286,34 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(TypeError, compile, chr(0), 'f', 'exec') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') + # test the optimize argument + + codestr = '''def f(): + """doc""" + try: + assert False + except AssertionError: + return (True, f.__doc__) + else: + return (False, f.__doc__) + ''' + def f(): """doc""" + values = [(-1, __debug__, f.__doc__), + (0, True, 'doc'), + (1, False, 'doc'), + (2, False, None)] + for optval, debugval, docstring in values: + # test both direct compilation and compilation via AST + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, (debugval, docstring)) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 1955006fd0f..4246b2f5da4 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -88,6 +88,15 @@ class CompileallTests(unittest.TestCase): compileall.compile_file(data_file) self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__'))) + def test_optimize(self): + # make sure compiling with different optimization settings than the + # interpreter's creates the correct file names + optimize = 1 if __debug__ else 0 + compileall.compile_dir(self.directory, quiet=True, optimize=optimize) + cached = imp.cache_from_source(self.source_path, + debug_override=not optimize) + self.assertTrue(os.path.isfile(cached)) + class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 7f93b68e27a..a0367e188d1 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -654,6 +654,22 @@ class PyZipFileTests(unittest.TestCase): self.assertTrue('email/mime/text.pyo' in names or 'email/mime/text.pyc' in names) + def test_write_with_optimization(self): + import email + packagedir = os.path.dirname(email.__file__) + # use .pyc if running test in optimization mode, + # use .pyo if running test in debug mode + optlevel = 1 if __debug__ else 0 + ext = '.pyo' if optlevel == 1 else '.pyc' + + with TemporaryFile() as t, \ + zipfile.PyZipFile(t, "w", optimize=optlevel) as zipfp: + zipfp.writepy(packagedir) + + names = zipfp.namelist() + self.assertIn('email/__init__' + ext, names) + self.assertIn('email/mime/text' + ext, names) + def test_write_python_directory(self): os.mkdir(TESTFN2) try: diff --git a/Lib/zipfile.py b/Lib/zipfile.py index bfe41b7b1a3..35bba7323bd 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1295,6 +1295,12 @@ class ZipFile: class PyZipFile(ZipFile): """Class to create ZIP archives with Python library files and packages.""" + def __init__(self, file, mode="r", compression=ZIP_STORED, + allowZip64=False, optimize=-1): + ZipFile.__init__(self, file, mode=mode, compression=compression, + allowZip64=allowZip64) + self._optimize = optimize + def writepy(self, pathname, basename=""): """Add all files from "pathname" to the ZIP archive. @@ -1367,44 +1373,63 @@ class PyZipFile(ZipFile): archive name, compiling if necessary. For example, given /python/lib/string, return (/python/lib/string.pyc, string). """ + def _compile(file, optimize=-1): + import py_compile + if self.debug: + print("Compiling", file) + try: + py_compile.compile(file, doraise=True, optimize=optimize) + except py_compile.PyCompileError as error: + print(err.msg) + return False + return True + file_py = pathname + ".py" file_pyc = pathname + ".pyc" file_pyo = pathname + ".pyo" pycache_pyc = imp.cache_from_source(file_py, True) pycache_pyo = imp.cache_from_source(file_py, False) - if (os.path.isfile(file_pyo) and - os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime): - # Use .pyo file. - arcname = fname = file_pyo - elif (os.path.isfile(file_pyc) and - os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): - # Use .pyc file. - arcname = fname = file_pyc - elif (os.path.isfile(pycache_pyc) and - os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_pyc - arcname = file_pyc - elif (os.path.isfile(pycache_pyo) and - os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyo file, but write it to the legacy pyo - # file name in the archive. - fname = pycache_pyo - arcname = file_pyo - else: - # Compile py into PEP 3147 pyc file. - import py_compile - if self.debug: - print("Compiling", file_py) - try: - py_compile.compile(file_py, doraise=True) - except py_compile.PyCompileError as error: - print(err.msg) - fname = file_py + if self._optimize == -1: + # legacy mode: use whatever file is present + if (os.path.isfile(file_pyo) and + os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyo file. + arcname = fname = file_pyo + elif (os.path.isfile(file_pyc) and + os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyc file. + arcname = fname = file_pyc + elif (os.path.isfile(pycache_pyc) and + os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_pyc + arcname = file_pyc + elif (os.path.isfile(pycache_pyo) and + os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyo file, but write it to the legacy pyo + # file name in the archive. + fname = pycache_pyo + arcname = file_pyo else: - fname = (pycache_pyc if __debug__ else pycache_pyo) - arcname = (file_pyc if __debug__ else file_pyo) + # Compile py into PEP 3147 pyc file. + if _compile(file_py): + fname = (pycache_pyc if __debug__ else pycache_pyo) + arcname = (file_pyc if __debug__ else file_pyo) + else: + fname = arcname = file_py + else: + # new mode: use given optimization level + if self._optimize == 0: + fname = pycache_pyc + arcname = file_pyc + else: + fname = pycache_pyo + arcname = file_pyo + if not (os.path.isfile(fname) and + os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): + if not _compile(file_py, optimize=self._optimize): + fname = arcname = file_py archivename = os.path.split(arcname)[1] if basename: archivename = "%s/%s" % (basename, archivename) diff --git a/Misc/NEWS b/Misc/NEWS index 576331b4f73..c2f288d6dfa 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.2 Beta 1? Core and Builtins ----------------- +- Provide an *optimize* parameter in the built-in compile() function. + - Fixed several corner case issues on os.stat/os.lstat related to reparse points. (Windows) @@ -40,6 +42,9 @@ Core and Builtins Library ------- +- Provide an interface to set the optimization level of compilation in + py_compile, compileall and zipfile.PyZipFile. + - Issue #7904: Changes to urllib.parse.urlsplit to handle schemes as defined by RFC3986. Anything before :// is considered a scheme and is followed by an authority (or netloc) and by '/' led path, which is optional. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 765464a73e9..f9b3202d54e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -543,19 +543,20 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) int mode = -1; int dont_inherit = 0; int supplied_flags = 0; + int optimize = -1; int is_ast; PyCompilerFlags cf; PyObject *cmd; static char *kwlist[] = {"source", "filename", "mode", "flags", - "dont_inherit", NULL}; + "dont_inherit", "optimize", NULL}; int start[] = {Py_file_input, Py_eval_input, Py_single_input}; PyObject *result; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&s|ii:compile", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&s|iii:compile", kwlist, &cmd, PyUnicode_FSConverter, &filename_obj, &startstr, &supplied_flags, - &dont_inherit)) + &dont_inherit, &optimize)) return NULL; filename = PyBytes_AS_STRING(filename_obj); @@ -570,6 +571,12 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) } /* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */ + if (optimize < -1 || optimize > 2) { + PyErr_SetString(PyExc_ValueError, + "compile(): invalid optimize value"); + goto error; + } + if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); } @@ -604,8 +611,8 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) PyArena_Free(arena); goto error; } - result = (PyObject*)PyAST_Compile(mod, filename, - &cf, arena); + result = (PyObject*)PyAST_CompileEx(mod, filename, + &cf, optimize, arena); PyArena_Free(arena); } goto finally; @@ -615,7 +622,7 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds) if (str == NULL) goto error; - result = Py_CompileStringFlags(str, filename, start[mode], &cf); + result = Py_CompileStringExFlags(str, filename, start[mode], &cf, optimize); goto finally; error: diff --git a/Python/compile.c b/Python/compile.c index dfb2c9bcf7f..1d6e38c22e3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -139,6 +139,7 @@ struct compiler { PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; + int c_optimize; /* optimization level */ int c_interactive; /* true if in interactive mode */ int c_nestlevel; @@ -175,7 +176,7 @@ static void compiler_pop_fblock(struct compiler *, enum fblocktype, static int compiler_in_loop(struct compiler *); static int inplace_binop(struct compiler *, operator_ty); -static int expr_constant(expr_ty e); +static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty); static int compiler_call_helper(struct compiler *c, int n, @@ -254,8 +255,8 @@ compiler_init(struct compiler *c) } PyCodeObject * -PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, - PyArena *arena) +PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, + int optimize, PyArena *arena) { struct compiler c; PyCodeObject *co = NULL; @@ -283,6 +284,7 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, c.c_future->ff_features = merged; flags->cf_flags = merged; c.c_flags = flags; + c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize; c.c_nestlevel = 0; c.c_st = PySymtable_Build(mod, filename, c.c_future); @@ -1149,7 +1151,7 @@ compiler_body(struct compiler *c, asdl_seq *stmts) if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); - if (compiler_isdocstring(st) && Py_OptimizeFlag < 2) { + if (compiler_isdocstring(st) && c->c_optimize < 2) { /* don't generate docstrings if -OO */ i = 1; VISIT(c, expr, st->v.Expr.value); @@ -1463,7 +1465,7 @@ compiler_function(struct compiler *c, stmt_ty s) st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0); docstring = compiler_isdocstring(st); - if (docstring && Py_OptimizeFlag < 2) + if (docstring && c->c_optimize < 2) first_const = st->v.Expr.value->v.Str.s; if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); @@ -1697,7 +1699,7 @@ compiler_if(struct compiler *c, stmt_ty s) if (end == NULL) return 0; - constant = expr_constant(s->v.If.test); + constant = expr_constant(c, s->v.If.test); /* constant = 0: "if 0" * constant = 1: "if 1", "if 2", ... * constant = -1: rest */ @@ -1759,7 +1761,7 @@ static int compiler_while(struct compiler *c, stmt_ty s) { basicblock *loop, *orelse, *end, *anchor = NULL; - int constant = expr_constant(s->v.While.test); + int constant = expr_constant(c, s->v.While.test); if (constant == 0) { if (s->v.While.orelse) @@ -2211,7 +2213,7 @@ compiler_assert(struct compiler *c, stmt_ty s) static PyObject *assertion_error = NULL; basicblock *end; - if (Py_OptimizeFlag) + if (c->c_optimize) return 1; if (assertion_error == NULL) { assertion_error = PyUnicode_InternFromString("AssertionError"); @@ -3011,7 +3013,7 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k) */ static int -expr_constant(expr_ty e) +expr_constant(struct compiler *c, expr_ty e) { char *id; switch (e->kind) { @@ -3029,7 +3031,7 @@ expr_constant(expr_ty e) if (strcmp(id, "False") == 0) return 0; if (strcmp(id, "None") == 0) return 0; if (strcmp(id, "__debug__") == 0) - return ! Py_OptimizeFlag; + return ! c->c_optimize; /* fall through */ default: return -1; @@ -4080,3 +4082,13 @@ assemble(struct compiler *c, int addNone) assemble_free(&a); return co; } + +#undef PyAST_Compile +PyAPI_FUNC(PyCodeObject *) +PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, + PyArena *arena) +{ + return PyAST_CompileEx(mod, filename, flags, -1, arena); +} + + diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 3f6385d455c..f7335a2b21d 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1793,8 +1793,8 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, } PyObject * -Py_CompileStringFlags(const char *str, const char *filename, int start, - PyCompilerFlags *flags) +Py_CompileStringExFlags(const char *str, const char *filename, int start, + PyCompilerFlags *flags, int optimize) { PyCodeObject *co; mod_ty mod; @@ -1812,7 +1812,7 @@ Py_CompileStringFlags(const char *str, const char *filename, int start, PyArena_Free(arena); return result; } - co = PyAST_Compile(mod, filename, flags, arena); + co = PyAST_CompileEx(mod, filename, flags, optimize, arena); PyArena_Free(arena); return (PyObject *)co; } @@ -2450,7 +2450,15 @@ PyRun_SimpleString(const char *s) PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { - return Py_CompileStringFlags(str, p, s, NULL); + return Py_CompileStringExFlags(str, p, s, NULL, -1); +} + +#undef Py_CompileStringFlags +PyAPI_FUNC(PyObject *) +Py_CompileStringFlags(const char *str, const char *p, int s, + PyCompilerFlags *flags) +{ + return Py_CompileStringExFlags(str, p, s, flags, -1); } #undef PyRun_InteractiveOne