bpo-35596: Use unchecked PYCs for the embeddable distro to avoid zipimport restrictions (GH-11465)

Also adds extra steps to the CI build for Windows on Azure Pipelines to validate that the various layouts at least execute.
This commit is contained in:
Steve Dower 2019-01-08 02:38:01 -08:00 committed by GitHub
parent c24c6c2c93
commit 872bd2b57c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 36 additions and 7 deletions

View File

@ -134,3 +134,13 @@ jobs:
steps:
- template: ./windows-steps.yml
- template: ./windows-layout-steps.yml
parameters:
kind: nuget
- template: ./windows-layout-steps.yml
parameters:
kind: embed
- template: ./windows-layout-steps.yml
parameters:
kind: appx

View File

@ -0,0 +1,11 @@
parameters:
kind: nuget
extraOpts: --precompile
steps:
- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
displayName: Create ${{ parameters['kind'] }} layout
- script: .\python.exe -m test.pythoninfo
workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
displayName: Show layout info (${{ parameters['kind'] }})

View File

@ -0,0 +1,2 @@
Use unchecked PYCs for the embeddable distro to avoid zipimport
restrictions.

View File

@ -240,12 +240,18 @@ def get_layout(ns):
yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
def _compile_one_py(src, dest, name, optimize):
def _compile_one_py(src, dest, name, optimize, checked=True):
import py_compile
if dest is not None:
dest = str(dest)
mode = (
py_compile.PycInvalidationMode.CHECKED_HASH
if checked
else py_compile.PycInvalidationMode.UNCHECKED_HASH
)
try:
return Path(
py_compile.compile(
@ -254,7 +260,7 @@ def _compile_one_py(src, dest, name, optimize):
str(name),
doraise=True,
optimize=optimize,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
invalidation_mode=mode,
)
)
except py_compile.PyCompileError:
@ -262,16 +268,16 @@ def _compile_one_py(src, dest, name, optimize):
return None
def _py_temp_compile(src, ns, dest_dir=None):
def _py_temp_compile(src, ns, dest_dir=None, checked=True):
if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
return None
dest = (dest_dir or ns.temp) / (src.stem + ".py")
return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2)
return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
def _write_to_zip(zf, dest, src, ns):
pyc = _py_temp_compile(src, ns)
def _write_to_zip(zf, dest, src, ns, checked=True):
pyc = _py_temp_compile(src, ns, checked=checked)
if pyc:
try:
zf.write(str(pyc), dest.with_suffix(".pyc"))
@ -321,7 +327,7 @@ def generate_source_files(ns):
ns.temp.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for dest, src in get_lib_layout(ns):
_write_to_zip(zf, dest, src, ns)
_write_to_zip(zf, dest, src, ns, checked=False)
if ns.include_underpth:
log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)