Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-113258: Write frozen modules to the build tree on Windows (GH-113303) 

This ensures the source directory is not modified at build time, and different builds (e.g. different versions or GIL vs no-GIL) do not have conflicts.
This commit is contained in:
Itamar Oren 2024-01-03 09:30:20 -08:00 committed by GitHub
parent 7d01fb4808
commit 178919cf21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS.d/next/Build/2023-12-23-09-35-48.gh-issue-113258.GlsAyH.rst Normal file
View File

@ -0,0 +1,2 @@
Changed the Windows build to write out generated frozen modules into the
build tree instead of the source tree.

98
PCbuild/_freeze_module.vcxproj
View File

@ -266,117 +266,117 @@
<None Include="..\Lib\importlib\_bootstrap.py">
<ModName>importlib._bootstrap</ModName>
<IntFile>$(IntDir)importlib._bootstrap.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h</OutFile>
</None>
<None Include="..\Lib\importlib\_bootstrap_external.py">
<ModName>importlib._bootstrap_external</ModName>
<IntFile>$(IntDir)importlib._bootstrap_external.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
</None>
<None Include="..\Lib\zipimport.py">
<ModName>zipimport</ModName>
<IntFile>$(IntDir)zipimport.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h</OutFile>
</None>
<None Include="..\Lib\abc.py">
<ModName>abc</ModName>
<IntFile>$(IntDir)abc.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h</OutFile>
</None>
<None Include="..\Lib\codecs.py">
<ModName>codecs</ModName>
<IntFile>$(IntDir)codecs.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h</OutFile>
</None>
<None Include="..\Lib\io.py">
<ModName>io</ModName>
<IntFile>$(IntDir)io.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\io.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\io.h</OutFile>
</None>
<None Include="..\Lib\_collections_abc.py">
<ModName>_collections_abc</ModName>
<IntFile>$(IntDir)_collections_abc.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\_collections_abc.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h</OutFile>
</None>
<None Include="..\Lib\_sitebuiltins.py">
<ModName>_sitebuiltins</ModName>
<IntFile>$(IntDir)_sitebuiltins.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\_sitebuiltins.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h</OutFile>
</None>
<None Include="..\Lib\genericpath.py">
<ModName>genericpath</ModName>
<IntFile>$(IntDir)genericpath.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\genericpath.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h</OutFile>
</None>
<None Include="..\Lib\ntpath.py">
<ModName>ntpath</ModName>
<IntFile>$(IntDir)ntpath.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\ntpath.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h</OutFile>
</None>
<None Include="..\Lib\posixpath.py">
<ModName>posixpath</ModName>
<IntFile>$(IntDir)posixpath.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h</OutFile>
</None>
<None Include="..\Lib\os.py">
<ModName>os</ModName>
<IntFile>$(IntDir)os.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\os.h</OutFile>
</None>
<None Include="..\Lib\site.py">
<ModName>site</ModName>
<IntFile>$(IntDir)site.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\site.h</OutFile>
</None>
<None Include="..\Lib\stat.py">
<ModName>stat</ModName>
<IntFile>$(IntDir)stat.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\stat.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h</OutFile>
</None>
<None Include="..\Lib\importlib\util.py">
<ModName>importlib.util</ModName>
<IntFile>$(IntDir)importlib.util.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\importlib.util.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h</OutFile>
</None>
<None Include="..\Lib\importlib\machinery.py">
<ModName>importlib.machinery</ModName>
<IntFile>$(IntDir)importlib.machinery.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\importlib.machinery.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h</OutFile>
</None>
<None Include="..\Lib\runpy.py">
<ModName>runpy</ModName>
<IntFile>$(IntDir)runpy.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\runpy.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h</OutFile>
</None>
<None Include="..\Lib\__hello__.py">
<ModName>__hello__</ModName>
<IntFile>$(IntDir)__hello__.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\__hello__.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h</OutFile>
</None>
<None Include="..\Lib\__phello__\__init__.py">
<ModName>__phello__</ModName>
<IntFile>$(IntDir)__phello__.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h</OutFile>
</None>
<None Include="..\Lib\__phello__\ham\__init__.py">
<ModName>__phello__.ham</ModName>
<IntFile>$(IntDir)__phello__.ham.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h</OutFile>
</None>
<None Include="..\Lib\__phello__\ham\eggs.py">
<ModName>__phello__.ham.eggs</ModName>
<IntFile>$(IntDir)__phello__.ham.eggs.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
</None>
<None Include="..\Lib\__phello__\spam.py">
<ModName>__phello__.spam</ModName>
<IntFile>$(IntDir)__phello__.spam.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\__phello__.spam.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h</OutFile>
</None>
<None Include="..\Tools\freeze\flag.py">
<ModName>frozen_only</ModName>
<IntFile>$(IntDir)frozen_only.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\frozen_only.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h</OutFile>
</None>
<!-- END frozen modules -->
</ItemGroup>
@ -385,34 +385,34 @@
<GetPath Include="..\Modules\getpath.py">
<ModName>getpath</ModName>
<IntFile>$(IntDir)getpath.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
<OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\getpath.h</OutFile>
</GetPath>
</ItemGroup>
<ItemGroup>
<!-- BEGIN freeze mappings -->
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\abc.h" FrozenId="abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\codecs.h" FrozenId="codecs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\io.h" FrozenId="io" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\os.h" FrozenId="os" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\site.h" FrozenId="site" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\stat.h" FrozenId="stat" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\runpy.h" FrozenId="runpy" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h" FrozenId="abc" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h" FrozenId="codecs" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\io.h" FrozenId="io" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\os.h" FrozenId="os" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\site.h" FrozenId="site" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h" FrozenId="stat" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h" FrozenId="runpy" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
<FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
<!-- END freeze mappings -->
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
@ -484,7 +484,7 @@ $(IntDir)\deepfreeze_mappings.txt
Overwrite="true"
Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" />
<!-- BEGIN deepfreeze rule -->
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(PySourcePath)Python\deepfreeze\deepfreeze.c"' />
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(GeneratedFrozenModulesDir)deepfreeze.c"' />
<!-- END deepfreeze rule -->
</Target>
<Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">
@ -493,7 +493,7 @@ $(IntDir)\deepfreeze_mappings.txt
<Clean Include="%(None.OutFile)" />
<Clean Include="%(GetPath.IntFile)" />
<Clean Include="%(GetPath.OutFile)" />
<Clean Include="$(PySourcePath)Python\deepfreeze\deepfreeze.c" />
<Clean Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />
</ItemGroup>
</Target>
</Project>

1
PCbuild/pyproject.props
View File

@ -12,6 +12,7 @@
<IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir>
<!-- pyconfig.h is updated by pythoncore.vcxproj, so it's always in pythoncore's IntDir -->
<GeneratedPyConfigDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\</GeneratedPyConfigDir>
<GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen_$(Configuration)\</GeneratedFrozenModulesDir>
<TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName>
<TargetName>$(TargetName)$(PyDebugExt)</TargetName>
<GenerateManifest>false</GenerateManifest>

8
PCbuild/pythoncore.vcxproj
View File

@ -111,6 +111,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\Modules\getpath.c">
<AdditionalIncludeDirectories>$(GeneratedFrozenModulesDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>
PREFIX=NULL;
EXEC_PREFIX=NULL;
@ -120,7 +121,6 @@
PLATLIBDIR="DLLs";
%(PreprocessorDefinitions)
</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(PySourcePath);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
</ItemGroup>
<ItemGroup>
@ -562,7 +562,9 @@
<ClCompile Include="..\Python\flowgraph.c" />
<ClCompile Include="..\Python\formatter_unicode.c" />
<ClCompile Include="..\Python\frame.c" />
<ClCompile Include="..\Python\frozen.c" />
<ClCompile Include="..\Python\frozen.c">
<AdditionalIncludeDirectories>$(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClCompile Include="..\Python\future.c" />
<ClCompile Include="..\Python\getargs.c" />
<ClCompile Include="..\Python\getcompiler.c" />
@ -617,7 +619,7 @@
</ItemGroup>
<ItemGroup>
<!-- BEGIN deepfreeze -->
<ClCompile Include="..\Python\deepfreeze\deepfreeze.c" />
<ClCompile Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />
<!-- END deepfreeze -->
</ItemGroup>
<ItemGroup Condition="$(IncludeExternals)">

8
Tools/build/freeze_modules.py
View File

@ -658,7 +658,7 @@ def regen_pcbuild(modules):
filterlines = []
corelines = []
deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
deepfreezerules = [f' <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\' />']
deepfreezerules = [f' <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(GeneratedFrozenModulesDir)deepfreeze.c"\' />']
deepfreezemappings = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
@ -667,15 +667,15 @@ def regen_pcbuild(modules):
projlines.append(f' <None Include="..\\{pyfile}">')
projlines.append(f' <ModName>{src.frozenid}</ModName>')
projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>')
projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>')
projlines.append(f' <OutFile>$(GeneratedFrozenModulesDir){header}</OutFile>')
projlines.append(f' </None>')
filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>')
deepfreezemappings.append(f' <FrozenModule Include="$(PySourcePath)\\{header}" FrozenId="{src.frozenid}" />\n')
deepfreezemappings.append(f' <FrozenModule Include="$(GeneratedFrozenModulesDir){header}" FrozenId="{src.frozenid}" />\n')
corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')
corelines.append(f' <ClCompile Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />')
print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):