bpo-46088: Automatically detect or install bootstrap Python runtime when building from Visual Studio (GH-30143)

Misc/NEWS.d/next/Build/2021-12-16-14-18-07.bpo-46088.8UUuAd.rst

@@ -0,0 +1,2 @@
+Automatically detect or install bootstrap Python runtime when building from
+Visual Studio.

PCbuild/_freeze_module.vcxproj

@@ -432,7 +432,10 @@
     <Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
              Condition="'@(_Updated)' != ''" Importance="high" />
   </Target>
-  <Target Name="_RebuildDeepFrozen" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'">
+  <Target Name="_RebuildDeepFrozen"
+          AfterTargets="_RebuildFrozen"
+          DependsOnTargets="FindPythonForBuild"
+          Condition="$(Configuration) != 'PGUpdate'">
     <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\scripts\deepfreeze.py" "%(None.OutFile)" "-m" "%(None.ModName)" -o "%(None.IntFile)"' />
 
     <Copy SourceFiles="%(None.IntFile)"

PCbuild/find_python.bat

@@ -6,6 +6,11 @@
 @rem changing any other persistent state.
 @rem
 
+@set _Py_D=%~dp0
+
+@rem First argument -q means only show the command in output
+@if '%1' EQU '-q' (shift && set _Py_Quiet=1)
+
 @rem No arguments provided means do full search
 @if '%1' EQU '' goto :begin_search
 
@@ -28,7 +33,7 @@
 @if NOT "%VIRTUAL_ENV%"=="" (set PYTHON="%VIRTUAL_ENV%\Scripts\python.exe") & (set _Py_Python_Source=found in virtual env) & goto :found
 
 @set _Py_EXTERNALS_DIR=%EXTERNALS_DIR%
-@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
+@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%_Py_D%\..\externals)
 
 @rem If we have Python in externals, use that one
 @if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
@@ -53,15 +58,18 @@
     @rem If it fails, retry with any available copy of Python
     @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
     @if errorlevel 1 (
-        @%_Py_HOST_PYTHON% -E "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+        @%_Py_HOST_PYTHON% -E "%_Py_D%\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
     )
 )
-@echo Installing Python via nuget...
+
+@if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget...
 @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
 @rem Quote it here; it's not quoted later because "py -x.y" wouldn't work
 @if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
 
 
+@set _Py_D=
+@set _Py_Quiet=
 @set _Py_Python_Source=
 @set _Py_EXTERNALS_DIR=
 @set _Py_NUGET=
@@ -70,7 +78,10 @@
 @exit /b 1
 
 :found
-@echo Using %PYTHON% (%_Py_Python_Source%)
+@if "%_Py_Quiet%"=="1" (@echo %PYTHON%) else @echo Using %PYTHON% (%_Py_Python_Source%)
+
+@set _Py_D=
+@set _Py_Quiet=
 @set _Py_Python_Source=
 @set _Py_EXTERNALS_DIR=
 @set _Py_NUGET=

PCbuild/pyproject.props

@@ -225,4 +225,17 @@ public override bool Execute() {
     <Warning Text="vcruntime*.dll not found under $(VCRedistDir)." Condition="@(VCRuntimeDLL) == ''" />
     <Message Text="VC Runtime DLL(s):%0A- @(VCRuntimeDLL,'%0A- ')" />
   </Target>
+
+  <Target Name="FindPythonForBuild" Condition="$(PythonForBuild) == ''">
+    <Exec Command="$(MSBuildThisFileDirectory)\find_python.bat -q"
+          EchoOff="true"
+          ConsoleToMsBuild="true">
+        <Output TaskParameter="ConsoleOutput" ItemName="_CmdExeLines" />
+    </Exec>
+    <PropertyGroup>
+      <PythonForBuild>@(_CmdExeLines)</PythonForBuild>
+    </PropertyGroup>
+    <Error Text="Failed to locate suitable Python runtime for building from source." Condition="$(PythonForBuild)==''" />
+    <Message Text="Using PythonForBuild=$(PythonForBuild)" Importance="high" />
+  </Target>
 </Project>

PCbuild/regen.targets

@@ -1,11 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Target Name="_SetPythonForBuild">
-    <PropertyGroup>
-      <PythonForBuild>$(PYTHON)</PythonForBuild>
-    </PropertyGroup>
-  </Target>
-
   <ItemGroup>
     <_PegenSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
     <_PegenOutputs Include="$(PySourcePath)Parser\parser.c" />
@@ -44,20 +38,26 @@
            AlwaysCreate="False" />
   </Target>
 
-  <Target Name="_RegenPegen" Inputs="@(_PegenSources)" Outputs="@(_PegenOutputs)">
+  <Target Name="_RegenPegen"
+          Inputs="@(_PegenSources)" Outputs="@(_PegenOutputs)"
+          DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_PegenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
     <!-- Specify python.gram with POSIX-like path because the argument gets written into the file verbatim -->
     <Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen -q c ./Grammar/python.gram Grammar\Tokens -o Parser\parser.c"
           WorkingDirectory="$(PySourcePath)" />
   </Target>
 
-  <Target Name="_RegenAST_H" Inputs="@(_ASTSources)" Outputs="@(_ASTOutputs)">
+  <Target Name="_RegenAST_H"
+          Inputs="@(_ASTSources)" Outputs="@(_ASTOutputs)"
+          DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_ASTOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
     <Exec Command="$(PythonForBuild) Parser\asdl_c.py Parser\Python.asdl @(_ASTOutputs->'%(Argument) &quot;%(Identity)&quot;',' ')"
           WorkingDirectory="$(PySourcePath)" />
   </Target>
 
-  <Target Name="_RegenOpcodes" Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)">
+  <Target Name="_RegenOpcodes"
+          Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
+          DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
     <Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
           WorkingDirectory="$(PySourcePath)" />
@@ -65,14 +65,18 @@
           WorkingDirectory="$(PySourcePath)" />
   </Target>
 
-  <Target Name="_RegenTokens" Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)">
+  <Target Name="_RegenTokens"
+          Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)"
+          DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_TokenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
     <Exec Command="$(PythonForBuild) Tools\scripts\generate_token.py %(_TokenOutputs.Format) Grammar\Tokens &quot;%(_TokenOutputs.Identity)&quot;"
           WorkingDirectory="$(PySourcePath)" />
     <Touch Files="@(_TokenOutputs)" />
   </Target>
 
-  <Target Name="_RegenKeywords" Inputs="@(_KeywordSources)" Outputs="@(_KeywordOutputs)">
+  <Target Name="_RegenKeywords"
+          Inputs="@(_KeywordSources)" Outputs="@(_KeywordOutputs)"
+          DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_KeywordOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
     <Exec Command="set PYTHONPATH=Tools\peg_generator%0D%0A$(PythonForBuild) -m pegen.keywordgen Grammar\python.gram Grammar\Tokens Lib\keyword.py"
           WorkingDirectory="$(PySourcePath)" />