traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-33499: Add PYTHONPYCACHEPREFIX env var for alt bytecode cache location. (GH-6834) 

In some development setups it is inconvenient or impossible to write bytecode
caches to the code tree, but the bytecode caches are still useful. The
PYTHONPYCACHEPREFIX environment variable allows specifying an alternate
location for cached bytecode files, within which a directory tree mirroring the code
tree will be created. This cache tree is then used (for both reading and writing)
instead of the local `__pycache__` subdirectory within each source directory.

Exposed at runtime as sys.pycache_prefix (defaulting to None), and can
be set from the CLI as "-X pycache_prefix=path".

Patch by Carl Meyer.
This commit is contained in:
Carl Meyer 2018-06-15 22:40:56 -06:00 committed by Nick Coghlan
parent 6868003514
commit b193fa996a
12 changed files with 1386 additions and 1087 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Doc/library/compileall.rst
View File

@ -109,6 +109,11 @@ There is no command-line option to control the optimization level used by the
:func:`compile` function, because the Python interpreter itself already
provides the option: :program:`python -O -m compileall`.
Similarly, the :func:`compile` function respects the :attr:`sys.pycache_prefix`
setting. The generated bytecode cache will only be useful if :func:`compile` is
run with the same :attr:`sys.pycache_prefix` (if any) that will be used at
runtime.
Public functions
----------------

20
Doc/library/sys.rst
View File

@ -209,6 +209,26 @@ always available.
yourself to control bytecode file generation.
.. data:: pycache_prefix
If this is set (not ``None``), Python will write bytecode-cache ``.pyc``
files to (and read them from) a parallel directory tree rooted at this
directory, rather than from ``__pycache__`` directories in the source code
tree. Any ``__pycache__`` directories in the source code tree will be ignored
and new `.pyc` files written within the pycache prefix. Thus if you use
:mod:`compileall` as a pre-build step, you must ensure you run it with the
same pycache prefix (if any) that you will use at runtime.
A relative path is interpreted relative to the current working directory.
This value is initially set based on the value of the :option:`-X`
``pycache_prefix=PATH`` command-line option or the
:envvar:`PYTHONPYCACHEPREFIX` environment variable (command-line takes
precedence). If neither are set, it is ``None``.
.. versionadded:: 3.8
.. function:: excepthook(type, value, traceback)
This function prints out a given traceback and exception to ``sys.stderr``.

16
Doc/using/cmdline.rst
View File

@ -442,6 +442,9 @@ Miscellaneous options
the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8
mode (even when it would otherwise activate automatically).
See :envvar:`PYTHONUTF8` for more details.
* ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel
tree rooted at the given directory instead of to the code tree. See also
:envvar:`PYTHONPYCACHEPREFIX`.
It also allows passing arbitrary values and retrieving them through the
:data:`sys._xoptions` dictionary.
@ -461,6 +464,9 @@ Miscellaneous options
.. versionadded:: 3.7
The ``-X importtime``, ``-X dev`` and ``-X utf8`` options.
.. versionadded:: 3.8
The ``-X pycache_prefix`` option.
Options you shouldn't use
~~~~~~~~~~~~~~~~~~~~~~~~~
@ -587,6 +593,16 @@ conflict.
specifying the :option:`-B` option.
.. envvar:: PYTHONPYCACHEPREFIX
If this is set, Python will write ``.pyc`` files in a mirror directory tree
at this path, instead of in ``__pycache__`` directories within the source
tree. This is equivalent to specifying the :option:`-X`
``pycache_prefix=PATH`` option.
.. versionadded:: 3.8
.. envvar:: PYTHONHASHSEED
If this variable is not set or set to ``random``, a random value is used

3
Include/pystate.h
View File

@ -1,4 +1,3 @@
/* Thread and interpreter state structures and their interfaces */
@ -44,6 +43,7 @@ typedef struct {
int coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */
int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
int utf8_mode; /* PYTHONUTF8, -X utf8; -1 means unknown */
wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
wchar_t *program_name; /* Program name, see also Py_GetProgramName() */
int argc; /* Number of command line arguments,
@ -101,6 +101,7 @@ typedef struct {
PyObject *warnoptions; /* sys.warnoptions list, can be NULL */
PyObject *xoptions; /* sys._xoptions dict, can be NULL */
PyObject *module_search_path; /* sys.path list */
PyObject *pycache_prefix; /* sys.pycache_prefix str, can be NULL */
} _PyMainInterpreterConfig;
#define _PyMainInterpreterConfig_INIT \

62
Lib/importlib/_bootstrap_external.py
View File

@ -102,6 +102,15 @@ def _path_isdir(path):
return _path_is_mode_type(path, 0o040000)
def _path_isabs(path):
"""Replacement for os.path.isabs.
Considers a Windows drive-relative path (no drive, but starts with slash) to
still be "absolute".
"""
return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon
def _write_atomic(path, data, mode=0o666):
"""Best-effort function to write data to a path atomically.
Be prepared to handle a FileExistsError if concurrent writing of the
@ -312,7 +321,33 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
if not optimization.isalnum():
raise ValueError('{!r} is not alphanumeric'.format(optimization))
almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
filename = almost_filename + BYTECODE_SUFFIXES[0]
if sys.pycache_prefix is not None:
# We need an absolute path to the py file to avoid the possibility of
# collisions within sys.pycache_prefix, if someone has two different
# `foo/bar.py` on their system and they import both of them using the
# same sys.pycache_prefix. Let's say sys.pycache_prefix is
# `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first
# make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative
# (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an
# unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`.
if not _path_isabs(head):
head = _path_join(_os.getcwd(), head)
# Strip initial drive from a Windows path. We know we have an absolute
# path here, so the second part of the check rules out a POSIX path that
# happens to contain a colon at the second character.
if head[1] == ':' and head[0] not in path_separators:
head = head[2:]
# Strip initial path separator from `head` to complete the conversion
# back to a root-relative path before joining.
return _path_join(
sys.pycache_prefix,
head.lstrip(path_separators),
filename,
)
return _path_join(head, _PYCACHE, filename)
def source_from_cache(path):
@ -328,23 +363,29 @@ def source_from_cache(path):
raise NotImplementedError('sys.implementation.cache_tag is None')
path = _os.fspath(path)
head, pycache_filename = _path_split(path)
head, pycache = _path_split(head)
if pycache != _PYCACHE:
raise ValueError('{} not bottom-level directory in '
'{!r}'.format(_PYCACHE, path))
found_in_pycache_prefix = False
if sys.pycache_prefix is not None:
stripped_path = sys.pycache_prefix.rstrip(path_separators)
if head.startswith(stripped_path + path_sep):
head = head[len(stripped_path):]
found_in_pycache_prefix = True
if not found_in_pycache_prefix:
head, pycache = _path_split(head)
if pycache != _PYCACHE:
raise ValueError(f'{_PYCACHE} not bottom-level directory in '
f'{path!r}')
dot_count = pycache_filename.count('.')
if dot_count not in {2, 3}:
raise ValueError('expected only 2 or 3 dots in '
'{!r}'.format(pycache_filename))
raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
elif dot_count == 3:
optimization = pycache_filename.rsplit('.', 2)[-2]
if not optimization.startswith(_OPT):
raise ValueError("optimization portion of filename does not start "
"with {!r}".format(_OPT))
f"with {_OPT!r}")
opt_level = optimization[len(_OPT):]
if not opt_level.isalnum():
raise ValueError("optimization level {!r} is not an alphanumeric "
"value".format(optimization))
raise ValueError(f"optimization level {optimization!r} is not an "
"alphanumeric value")
base_filename = pycache_filename.partition('.')[0]
return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
@ -1533,6 +1574,7 @@ def _setup(_bootstrap_module):
setattr(self_module, '_os', os_module)
setattr(self_module, 'path_sep', path_sep)
setattr(self_module, 'path_separators', ''.join(path_separators))
setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
# Directly load the _thread module (needed during bootstrap).
thread_module = _bootstrap._builtin_from_name('_thread')

26
Lib/test/test_cmd_line.py
View File

@ -519,6 +519,32 @@ class CmdLineTest(unittest.TestCase):
with self.subTest(envar_value=value):
assert_python_ok('-c', code, **env_vars)
def test_set_pycache_prefix(self):
# sys.pycache_prefix can be set from either -X pycache_prefix or
# PYTHONPYCACHEPREFIX env var, with the former taking precedence.
NO_VALUE = object() # `-X pycache_prefix` with no `=PATH`
cases = [
# (PYTHONPYCACHEPREFIX, -X pycache_prefix, sys.pycache_prefix)
(None, None, None),
('foo', None, 'foo'),
(None, 'bar', 'bar'),
('foo', 'bar', 'bar'),
('foo', '', None),
('foo', NO_VALUE, None),
]
for envval, opt, expected in cases:
exp_clause = "is None" if expected is None else f'== "{expected}"'
code = f"import sys; sys.exit(not sys.pycache_prefix {exp_clause})"
args = ['-c', code]
env = {} if envval is None else {'PYTHONPYCACHEPREFIX': envval}
if opt is NO_VALUE:
args[:0] = ['-X', 'pycache_prefix']
elif opt is not None:
args[:0] = ['-X', f'pycache_prefix={opt}']
with self.subTest(envval=envval, opt=opt):
with support.temp_cwd():
assert_python_ok(*args, **env)
def run_xdev(self, *args, check_exitcode=True, xdev=True):
env = dict(os.environ)
env.pop('PYTHONWARNINGS', None)

95
Lib/test/test_importlib/test_util.py
View File

@ -1,9 +1,10 @@
from . import util
from . import util
abc = util.import_importlib('importlib.abc')
init = util.import_importlib('importlib')
machinery = util.import_importlib('importlib.machinery')
importlib_util = util.import_importlib('importlib.util')
import contextlib
import importlib.util
import os
import pathlib
@ -12,6 +13,7 @@ import sys
from test import support
import types
import unittest
import unittest.mock
import warnings
@ -557,8 +559,8 @@ class PEP3147Tests:
tag = sys.implementation.cache_tag
@unittest.skipUnless(sys.implementation.cache_tag is not None,
'requires sys.implementation.cache_tag not be None')
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag not be None')
def test_cache_from_source(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyc file (i.e. under __pycache__).
@ -678,8 +680,8 @@ class PEP3147Tests:
self.util.cache_from_source('\\foo\\bar\\baz/qux.py', optimization=''),
'\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
@unittest.skipUnless(sys.implementation.cache_tag is not None,
'requires sys.implementation.cache_tag not be None')
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag not be None')
def test_source_from_cache_path_like_arg(self):
path = pathlib.PurePath('foo', 'bar', 'baz', 'qux.py')
expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
@ -687,9 +689,8 @@ class PEP3147Tests:
self.assertEqual(self.util.cache_from_source(path, optimization=''),
expect)
@unittest.skipUnless(sys.implementation.cache_tag is not None,
'requires sys.implementation.cache_tag to not be '
'None')
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_source_from_cache(self):
# Given the path to a PEP 3147 defined .pyc file, return the path to
# its source. This tests the good path.
@ -749,15 +750,87 @@ class PEP3147Tests:
with self.assertRaises(ValueError):
self.util.source_from_cache(path)
@unittest.skipUnless(sys.implementation.cache_tag is not None,
'requires sys.implementation.cache_tag to not be '
'None')
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_source_from_cache_path_like_arg(self):
path = pathlib.PurePath('foo', 'bar', 'baz', '__pycache__',
'qux.{}.pyc'.format(self.tag))
expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
self.assertEqual(self.util.source_from_cache(path), expect)
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_cache_from_source_respects_pycache_prefix(self):
# If pycache_prefix is set, cache_from_source will return a bytecode
# path inside that directory (in a subdirectory mirroring the .py file's
# path) rather than in a __pycache__ dir next to the py file.
pycache_prefixes = [
os.path.join(os.path.sep, 'tmp', 'bytecode'),
os.path.join(os.path.sep, 'tmp', '\u2603'), # non-ASCII in path!
os.path.join(os.path.sep, 'tmp', 'trailing-slash') + os.path.sep,
]
drive = ''
if os.name == 'nt':
drive = 'C:'
pycache_prefixes = [
f'{drive}{prefix}' for prefix in pycache_prefixes]
pycache_prefixes += [r'\\?\C:\foo', r'\\localhost\c$\bar']
for pycache_prefix in pycache_prefixes:
with self.subTest(path=pycache_prefix):
path = drive + os.path.join(
os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
expect = os.path.join(
pycache_prefix, 'foo', 'bar', 'baz',
'qux.{}.pyc'.format(self.tag))
with util.temporary_pycache_prefix(pycache_prefix):
self.assertEqual(
self.util.cache_from_source(path, optimization=''),
expect)
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_cache_from_source_respects_pycache_prefix_relative(self):
# If the .py path we are given is relative, we will resolve to an
# absolute path before prefixing with pycache_prefix, to avoid any
# possible ambiguity.
pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
path = os.path.join('foo', 'bar', 'baz', 'qux.py')
root = os.path.splitdrive(os.getcwd())[0] + os.path.sep
expect = os.path.join(
pycache_prefix,
os.path.relpath(os.getcwd(), root),
'foo', 'bar', 'baz', f'qux.{self.tag}.pyc')
with util.temporary_pycache_prefix(pycache_prefix):
self.assertEqual(
self.util.cache_from_source(path, optimization=''),
expect)
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_source_from_cache_inside_pycache_prefix(self):
# If pycache_prefix is set and the cache path we get is inside it,
# we return an absolute path to the py file based on the remainder of
# the path within pycache_prefix.
pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
path = os.path.join(pycache_prefix, 'foo', 'bar', 'baz',
f'qux.{self.tag}.pyc')
expect = os.path.join(os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
with util.temporary_pycache_prefix(pycache_prefix):
self.assertEqual(self.util.source_from_cache(path), expect)
@unittest.skipIf(sys.implementation.cache_tag is None,
'requires sys.implementation.cache_tag to not be None')
def test_source_from_cache_outside_pycache_prefix(self):
# If pycache_prefix is set but the cache path we get is not inside
# it, just ignore it and handle the cache path according to the default
# behavior.
pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
path = os.path.join('foo', 'bar', 'baz', '__pycache__',
f'qux.{self.tag}.pyc')
expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
with util.temporary_pycache_prefix(pycache_prefix):
self.assertEqual(self.util.source_from_cache(path), expect)
(Frozen_PEP3147Tests,
Source_PEP3147Tests

11
Lib/test/test_importlib/util.py
View File

@ -319,6 +319,17 @@ def ensure_bytecode_path(bytecode_path):
raise
@contextlib.contextmanager
def temporary_pycache_prefix(prefix):
"""Adjust and restore sys.pycache_prefix."""
_orig_prefix = sys.pycache_prefix
sys.pycache_prefix = prefix
try:
yield
finally:
sys.pycache_prefix = _orig_prefix
@contextlib.contextmanager
def create_modules(*names):
"""Temporarily create each named module with an attribute (named 'attr')

3
Misc/NEWS.d/next/Core and Builtins/2018-05-15-10-48-47.bpo-33499.uBEc06.rst Normal file
View File

@ -0,0 +1,3 @@
Add :envvar:`PYTHONPYCACHEPREFIX` environment variable and :option:`-X`
``pycache_prefix`` command-line option to set an alternate root directory for
writing module bytecode cache files.

51
Modules/main.c
View File

@ -144,7 +144,8 @@ static const char usage_6[] =
"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n"
" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n"
" locale coercion and locale compatibility warnings on stderr.\n"
"PYTHONDEVMODE: enable the development mode.\n";
"PYTHONDEVMODE: enable the development mode.\n"
"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n";
static void
pymain_usage(int error, const wchar_t* program)
@ -1675,6 +1676,37 @@ pymain_init_tracemalloc(_PyCoreConfig *config)
}
static _PyInitError
pymain_init_pycache_prefix(_PyCoreConfig *config)
{
wchar_t *env;
int res = config_get_env_var_dup(
&env, L"PYTHONPYCACHEPREFIX", "PYTHONPYCACHEPREFIX");
if (res < 0) {
return DECODE_LOCALE_ERR("PYTHONPYCACHEPREFIX", res);
} else if (env) {
config->pycache_prefix = env;
}
const wchar_t *xoption = config_get_xoption(config, L"pycache_prefix");
if (xoption) {
const wchar_t *sep = wcschr(xoption, L'=');
if (sep && wcslen(sep) > 1) {
config->pycache_prefix = _PyMem_RawWcsdup(sep + 1);
if (config->pycache_prefix == NULL) {
return _Py_INIT_NO_MEMORY();
}
} else {
// -X pycache_prefix= can cancel the env var
config->pycache_prefix = NULL;
}
}
return _Py_INIT_OK();
}
static void
get_env_flag(int *flag, const char *name)
{
@ -1868,6 +1900,12 @@ config_read_complex_options(_PyCoreConfig *config)
if (_Py_INIT_FAILED(err)) {
return err;
}
err = pymain_init_pycache_prefix(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
return _Py_INIT_OK();
}
@ -2238,6 +2276,7 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
LIST = NULL; \
} while (0)
CLEAR(config->pycache_prefix);
CLEAR(config->module_search_path_env);
CLEAR(config->home);
CLEAR(config->program_name);
@ -2302,6 +2341,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
COPY_ATTR(malloc_stats);
COPY_ATTR(utf8_mode);
COPY_STR_ATTR(pycache_prefix);
COPY_STR_ATTR(module_search_path_env);
COPY_STR_ATTR(home);
COPY_STR_ATTR(program_name);
@ -2337,6 +2377,7 @@ _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config)
Py_CLEAR(config->warnoptions);
Py_CLEAR(config->xoptions);
Py_CLEAR(config->module_search_path);
Py_CLEAR(config->pycache_prefix);
}
@ -2389,6 +2430,7 @@ _PyMainInterpreterConfig_Copy(_PyMainInterpreterConfig *config,
COPY_ATTR(warnoptions);
COPY_ATTR(xoptions);
COPY_ATTR(module_search_path);
COPY_ATTR(pycache_prefix);
#undef COPY_ATTR
return 0;
}
@ -2446,6 +2488,13 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config,
COPY_WSTRLIST(main_config->module_search_path,
config->nmodule_search_path, config->module_search_paths);
if (config->pycache_prefix != NULL) {
COPY_WSTR(pycache_prefix);
} else {
main_config->pycache_prefix = NULL;
}
}
return _Py_INIT_OK();

2175
Python/importlib_external.h generated
View File

File diff suppressed because it is too large Load Diff

6
Python/sysmodule.c
View File

@ -2458,6 +2458,12 @@ _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *config)
SET_SYS_FROM_STRING_BORROW("exec_prefix", config->exec_prefix);
SET_SYS_FROM_STRING_BORROW("base_exec_prefix", config->base_exec_prefix);
if (config->pycache_prefix != NULL) {
SET_SYS_FROM_STRING_BORROW("pycache_prefix", config->pycache_prefix);
} else {
PyDict_SetItemString(sysdict, "pycache_prefix", Py_None);
}
if (config->argv != NULL) {
SET_SYS_FROM_STRING_BORROW("argv", config->argv);
}