traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-36500: Add --regen option to PCbuild/build.bat so Windows users can regen grammar, opcodes, tokens and symbols (GH-12654)

This commit is contained in:
Anthony Shaw 2019-12-18 08:21:20 +11:00 committed by Steve Dower
parent 9707e8e22d
commit 9e36589d49
3 changed files with 243 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS.d/next/Build/2019-04-02-01-59-26.bpo-36500.fyG6_U.rst Normal file
View File

@ -0,0 +1,2 @@
Added an optional "regen" project to the Visual Studio solution that will
regenerate all grammar, tokens, and opcodes.

17
PCbuild/build.bat
View File

@ -32,6 +32,7 @@ echo. automatically by the pythoncore project)
echo. --pgo Build with Profile-Guided Optimization. This flag echo. --pgo Build with Profile-Guided Optimization. This flag
echo. overrides -c and -d echo. overrides -c and -d
echo. --test-marker Enable the test marker within the build. echo. --test-marker Enable the test marker within the build.
echo. --regen Regenerate all opcodes, grammar and tokens
echo. echo.
echo.Available flags to avoid building certain modules. echo.Available flags to avoid building certain modules.
echo.These flags have no effect if '-e' is not given: echo.These flags have no effect if '-e' is not given:
@ -76,7 +77,8 @@ if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts
if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts
if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts
if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
if "%~1"=="-V" shift & goto :Version if "%~1"=="-V" shift & goto Version
if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts
rem These use the actual property names used by MSBuild. We could just let rem These use the actual property names used by MSBuild. We could just let
rem them in through the environment, but we specify them on the command line rem them in through the environment, but we specify them on the command line
rem anyway for visibility so set defaults after this rem anyway for visibility so set defaults after this
@ -154,9 +156,22 @@ echo on
/p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^
%1 %2 %3 %4 %5 %6 %7 %8 %9 %1 %2 %3 %4 %5 %6 %7 %8 %9
if "%Regen%"=="true" call :Regen
@echo off @echo off
exit /b %ERRORLEVEL% exit /b %ERRORLEVEL%
:Regen
echo on
call "%dir%find_msbuild.bat" %MSBUILD%
if not ERRORLEVEL 1 %MSBUILD% "%dir%regen.vcxproj" /t:%target% %parallel% %verbose%^
/p:IncludeExternals=%IncludeExternals%^
/p:Configuration=%conf% /p:Platform=%platf%^
/p:UseTestMarker=%UseTestMarker% %GITProperty%^
%1 %2 %3 %4 %5 %6 %7 %8 %9
@echo off
goto :eof
:Version :Version
rem Display the current build version information rem Display the current build version information
call "%dir%find_msbuild.bat" %MSBUILD% call "%dir%find_msbuild.bat" %MSBUILD%

225
PCbuild/regen.vcxproj Normal file
View File

@ -0,0 +1,225 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|ARM">
<Configuration>Debug</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|ARM">
<Configuration>PGInstrument</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|Win32">
<Configuration>PGInstrument</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGInstrument|x64">
<Configuration>PGInstrument</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|ARM">
<Configuration>PGUpdate</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|Win32">
<Configuration>PGUpdate</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PGUpdate|x64">
<Configuration>PGUpdate</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|ARM">
<Configuration>Release</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{21CF2108-2CC9-4005-A6ED-B7965ADE3854}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>regen</RootNamespace>
<SupportPGO>false</SupportPGO>
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="python.props" />
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Label="Configuration">
<ConfigurationType>Utility</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|ARM'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|Win32'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|ARM'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|Win32'">
<PlatformToolset>v142</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="pyproject.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\Programs\_freeze_importlib.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="pythoncore.vcxproj">
<Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
<Private>true</Private>
<ReferenceOutputAssembly>true</ReferenceOutputAssembly>
<CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
<LinkLibraryDependencies>true</LinkLibraryDependencies>
<UseLibraryDependencyInputs>false</UseLibraryDependencyInputs>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="..\Grammar\Grammar">
</None>
<None Include="..\Grammar\Tokens">
</None>
<None Include="..\Include\graminit.h">
</None>
<None Include="..\Python\graminit.c">
</None>
<None Include="..\Include\token.h">
</None>
<None Include="..\Include\opcode.h">
</None>
<None Include="..\Include\Python-ast.h">
</None>
<None Include="..\Python\Python-ast.c">
</None>
<None Include="..\Lib\keyword.py">
</None>
<None Include="..\Lib\symbol.py">
</None>
<None Include="..\Doc\library\token-list.inc">
</None>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
<Target Name="_RegenGrammar" BeforeTargets="Build">
<!-- Regenerate Include/graminit.h and Python/graminit.c from Grammar/Grammar using pgen-->
<Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)graminit.h&quot; &quot;$(IntDir)graminit.c&quot;" />
<Copy SourceFiles="$(IntDir)graminit.h" DestinationFiles="$(PySourcePath)Include\graminit.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy>
<Copy SourceFiles="$(IntDir)graminit.c" DestinationFiles="$(PySourcePath)Python\graminit.c">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
</Copy>
<Warning Text="Grammar updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
</Target>
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
<!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy>
<Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
</Target>
<Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
<!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy>
<Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
</Target>
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_opcode_h.py &quot;$(PySourcePath)Lib\opcode.py&quot; &quot;$(IntDir)opcode.h&quot;" />
<Copy SourceFiles="$(IntDir)opcode.h" DestinationFiles="$(PySourcePath)Include\opcode.h">
<Output TaskParameter="CopiedFiles" ItemName="_Updated" />
</Copy>
<Warning Text="Opcodes updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
</Target>
<Target Name="_RegenTokens" AfterTargets="_RegenOpcodes">
<!-- Regenerate Doc/library/token-list.inc from Grammar/Tokens using Tools/scripts/generate_token.py-->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py rst &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Doc\library\token-list.inc&quot;" />
<!-- Regenerate Include/token.h from Grammar/Tokens using Tools/scripts/generate_token.py-->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py h &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Include\token.h&quot;" />
<!-- Regenerate Parser/token.c from Grammar/Tokens using Tools/scripts/generate_token.py-->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py c &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Parser\token.c&quot;" />
<!-- Regenerate Lib/token.py from Grammar/Tokens using Tools/scripts/generate_token.py -->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py py &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Lib\token.py&quot;" />
</Target>
<Target Name="_RegenKeywords" AfterTargets="_RegenTokens">
<!-- Regenerate Lib/keyword.py from Grammar/Grammar and Grammar/Tokens using Parser/pgen-->
<Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen.keywordgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)keyword.py&quot;" />
<Copy SourceFiles="$(IntDir)keyword.py" DestinationFiles="$(PySourcePath)Lib\keyword.py">
<Output TaskParameter="CopiedFiles" ItemName="_Updated" />
</Copy>
<Warning Text="Keywords updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
</Target>
<Target Name="_RegenSymbols" AfterTargets="_RegenKeywords">
<!-- Regenerate Lib/symbol.py from Include/graminit.h using Tools/scripts/generate_symbol_py.py-->
<Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_symbol_py.py &quot;$(PySourcePath)Include\graminit.h&quot; &quot;$(PySourcePath)Lib\symbol.py&quot;" />
</Target>
<Target Name="_CleanFiles" BeforeTargets="CoreClean">
<ItemGroup>
<Clean Include="$(IntDir)keyword.py" />
<Clean Include="$(IntDir)opcode.h" />
<Clean Include="$(IntDir)Python-ast.c" />
<Clean Include="$(IntDir)Python-ast.h" />
<Clean Include="$(IntDir)graminit.h.new" />
<Clean Include="$(IntDir)graminit.c.new" />
</ItemGroup>
</Target>
</Project>