mirror of https://github.com/python/cpython
gh-109615: Fix test_tools.test_freeze SRCDIR (#109935)
Fix copy_source_tree() function of test_tools.test_freeze: * Don't copy SRC_DIR/build/ anymore. This directory is modified by other tests running in parallel. * Add test.support.copy_python_src_ignore(). * Use sysconfig to get the source directory. * Use sysconfig.get_config_var() to get CONFIG_ARGS variable.
This commit is contained in:
parent
0e28d0f7a1
commit
1512d6c6ee
|
@ -355,7 +355,7 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath:
|
|||
if not support.is_wasi:
|
||||
tmp_dir = sysconfig.get_config_var('abs_builddir')
|
||||
if tmp_dir is None:
|
||||
# bpo-30284: On Windows, only srcdir is available. Using
|
||||
# gh-74470: On Windows, only srcdir is available. Using
|
||||
# abs_builddir mostly matters on UNIX when building Python
|
||||
# out of the source tree, especially when the source tree
|
||||
# is read only.
|
||||
|
|
|
@ -2565,3 +2565,30 @@ def without_optimizer(func):
|
|||
finally:
|
||||
_testinternalcapi.set_optimizer(save_opt)
|
||||
return wrapper
|
||||
|
||||
|
||||
_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({
|
||||
# SRC_DIR/.git
|
||||
'.git',
|
||||
# ignore all __pycache__/ sub-directories
|
||||
'__pycache__',
|
||||
})
|
||||
|
||||
# Ignore function for shutil.copytree() to copy the Python source code.
|
||||
def copy_python_src_ignore(path, names):
|
||||
ignored = _BASE_COPY_SRC_DIR_IGNORED_NAMES
|
||||
if os.path.basename(path) == 'Doc':
|
||||
ignored |= {
|
||||
# SRC_DIR/Doc/build/
|
||||
'build',
|
||||
# SRC_DIR/Doc/venv/
|
||||
'venv',
|
||||
}
|
||||
|
||||
# check if we are at the root of the source code
|
||||
elif 'Modules' in names:
|
||||
ignored |= {
|
||||
# SRC_DIR/build/
|
||||
'build',
|
||||
}
|
||||
return ignored
|
||||
|
|
|
@ -7,6 +7,7 @@ import socket
|
|||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import sysconfig
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
@ -800,6 +801,27 @@ class TestSupport(unittest.TestCase):
|
|||
support.max_memuse = old_max_memuse
|
||||
support.real_max_memuse = old_real_max_memuse
|
||||
|
||||
def test_copy_python_src_ignore(self):
|
||||
src_dir = sysconfig.get_config_var('srcdir')
|
||||
src_dir = os.path.abspath(src_dir)
|
||||
|
||||
ignored = {'.git', '__pycache__'}
|
||||
|
||||
# Source code directory
|
||||
names = os.listdir(src_dir)
|
||||
self.assertEqual(support.copy_python_src_ignore(src_dir, names),
|
||||
ignored | {'build'})
|
||||
|
||||
# Doc/ directory
|
||||
path = os.path.join(src_dir, 'Doc')
|
||||
self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)),
|
||||
ignored | {'build', 'venv'})
|
||||
|
||||
# An other directory
|
||||
path = os.path.join(src_dir, 'Objects')
|
||||
self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)),
|
||||
ignored)
|
||||
|
||||
# XXX -follows a list of untested API
|
||||
# make_legacy_pyc
|
||||
# is_resource_enabled
|
||||
|
|
|
@ -21,7 +21,7 @@ from test.support import (captured_stdout, captured_stderr,
|
|||
skip_if_broken_multiprocessing_synchronize, verbose,
|
||||
requires_subprocess, is_emscripten, is_wasi,
|
||||
requires_venv_with_pip, TEST_HOME_DIR,
|
||||
requires_resource)
|
||||
requires_resource, copy_python_src_ignore)
|
||||
from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
|
||||
import unittest
|
||||
import venv
|
||||
|
@ -560,12 +560,6 @@ class BasicTest(BaseTest):
|
|||
stdlib_zip)
|
||||
additional_pythonpath_for_non_installed = []
|
||||
|
||||
# gh-109748: Don't copy __pycache__/ sub-directories, because they can
|
||||
# be modified by other Python tests running in parallel.
|
||||
ignored_names = {'__pycache__'}
|
||||
def ignore_pycache(src, names):
|
||||
return ignored_names
|
||||
|
||||
# Copy stdlib files to the non-installed python so venv can
|
||||
# correctly calculate the prefix.
|
||||
for eachpath in sys.path:
|
||||
|
@ -583,7 +577,7 @@ class BasicTest(BaseTest):
|
|||
shutil.copy(fn, libdir)
|
||||
elif os.path.isdir(fn):
|
||||
shutil.copytree(fn, os.path.join(libdir, name),
|
||||
ignore=ignore_pycache)
|
||||
ignore=copy_python_src_ignore)
|
||||
else:
|
||||
additional_pythonpath_for_non_installed.append(
|
||||
eachpath)
|
||||
|
|
|
@ -1,14 +1,15 @@
|
|||
import os
|
||||
import os.path
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sysconfig
|
||||
from test import support
|
||||
|
||||
|
||||
TESTS_DIR = os.path.dirname(__file__)
|
||||
TOOL_ROOT = os.path.dirname(TESTS_DIR)
|
||||
SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT))
|
||||
SRCDIR = os.path.abspath(sysconfig.get_config_var('srcdir'))
|
||||
|
||||
MAKE = shutil.which('make')
|
||||
FREEZE = os.path.join(TOOL_ROOT, 'freeze.py')
|
||||
|
@ -75,56 +76,17 @@ def ensure_opt(args, name, value):
|
|||
|
||||
|
||||
def copy_source_tree(newroot, oldroot):
|
||||
print(f'copying the source tree into {newroot}...')
|
||||
print(f'copying the source tree from {oldroot} to {newroot}...')
|
||||
if os.path.exists(newroot):
|
||||
if newroot == SRCDIR:
|
||||
raise Exception('this probably isn\'t what you wanted')
|
||||
shutil.rmtree(newroot)
|
||||
|
||||
def ignore_non_src(src, names):
|
||||
"""Turns what could be a 1000M copy into a 100M copy."""
|
||||
# Don't copy the ~600M+ of needless git repo metadata.
|
||||
# source only, ignore cached .pyc files.
|
||||
subdirs_to_skip = {'.git', '__pycache__'}
|
||||
if os.path.basename(src) == 'Doc':
|
||||
# Another potential ~250M+ of non test related data.
|
||||
subdirs_to_skip.add('build')
|
||||
subdirs_to_skip.add('venv')
|
||||
return subdirs_to_skip
|
||||
|
||||
shutil.copytree(oldroot, newroot, ignore=ignore_non_src)
|
||||
shutil.copytree(oldroot, newroot, ignore=support.copy_python_src_ignore)
|
||||
if os.path.exists(os.path.join(newroot, 'Makefile')):
|
||||
_run_quiet([MAKE, 'clean'], newroot)
|
||||
|
||||
|
||||
def get_makefile_var(builddir, name):
|
||||
regex = re.compile(rf'^{name} *=\s*(.*?)\s*$')
|
||||
filename = os.path.join(builddir, 'Makefile')
|
||||
try:
|
||||
infile = open(filename, encoding='utf-8')
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
with infile:
|
||||
for line in infile:
|
||||
m = regex.match(line)
|
||||
if m:
|
||||
value, = m.groups()
|
||||
return value or ''
|
||||
return None
|
||||
|
||||
|
||||
def get_config_var(builddir, name):
|
||||
python = os.path.join(builddir, 'python')
|
||||
if os.path.isfile(python):
|
||||
cmd = [python, '-c',
|
||||
f'import sysconfig; print(sysconfig.get_config_var("{name}"))']
|
||||
try:
|
||||
return _run_stdout(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
return get_makefile_var(builddir, name)
|
||||
|
||||
|
||||
##################################
|
||||
# freezing
|
||||
|
||||
|
@ -151,10 +113,8 @@ def prepare(script=None, outdir=None):
|
|||
|
||||
# Run configure.
|
||||
print(f'configuring python in {builddir}...')
|
||||
cmd = [
|
||||
os.path.join(srcdir, 'configure'),
|
||||
*shlex.split(get_config_var(SRCDIR, 'CONFIG_ARGS') or ''),
|
||||
]
|
||||
config_args = shlex.split(sysconfig.get_config_var('CONFIG_ARGS') or '')
|
||||
cmd = [os.path.join(srcdir, 'configure'), *config_args]
|
||||
ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache'))
|
||||
prefix = os.path.join(outdir, 'python-installation')
|
||||
ensure_opt(cmd, 'prefix', prefix)
|
||||
|
|
Loading…
Reference in New Issue