From 9e36589d49c1d6b06c0239fa69e8274d7e89e375 Mon Sep 17 00:00:00 2001
From: Anthony Shaw <anthony.p.shaw@gmail.com>
Date: Wed, 18 Dec 2019 08:21:20 +1100
Subject: [PATCH] bpo-36500: Add --regen option to PCbuild/build.bat so Windows
 users can regen grammar, opcodes, tokens and symbols (GH-12654)

---
 .../2019-04-02-01-59-26.bpo-36500.fyG6_U.rst  |   2 +
 PCbuild/build.bat                             |  17 +-
 PCbuild/regen.vcxproj                         | 225 ++++++++++++++++++
 3 files changed, 243 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Build/2019-04-02-01-59-26.bpo-36500.fyG6_U.rst
 create mode 100644 PCbuild/regen.vcxproj

diff --git a/Misc/NEWS.d/next/Build/2019-04-02-01-59-26.bpo-36500.fyG6_U.rst b/Misc/NEWS.d/next/Build/2019-04-02-01-59-26.bpo-36500.fyG6_U.rst
new file mode 100644
index 00000000000..cd230e4069e
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-04-02-01-59-26.bpo-36500.fyG6_U.rst
@@ -0,0 +1,2 @@
+Added an optional "regen" project to the Visual Studio solution that will
+regenerate all grammar, tokens, and opcodes.
diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index 623409c24ec..5bc5da9e1c9 100644
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -32,6 +32,7 @@ echo.      automatically by the pythoncore project)
 echo.  --pgo          Build with Profile-Guided Optimization.  This flag
 echo.                 overrides -c and -d
 echo.  --test-marker  Enable the test marker within the build.
+echo.  --regen        Regenerate all opcodes, grammar and tokens
 echo.
 echo.Available flags to avoid building certain modules.
 echo.These flags have no effect if '-e' is not given:
@@ -76,7 +77,8 @@ if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts
 if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts
 if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts
 if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
-if "%~1"=="-V" shift & goto :Version
+if "%~1"=="-V" shift & goto Version
+if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts
 rem These use the actual property names used by MSBuild.  We could just let
 rem them in through the environment, but we specify them on the command line
 rem anyway for visibility so set defaults after this
@@ -154,9 +156,22 @@ echo on
  /p:UseTestMarker=%UseTestMarker% %GITProperty%^
  %1 %2 %3 %4 %5 %6 %7 %8 %9
 
+if "%Regen%"=="true" call :Regen
 @echo off
 exit /b %ERRORLEVEL%
 
+:Regen
+echo on
+call "%dir%find_msbuild.bat" %MSBUILD%
+if not ERRORLEVEL 1 %MSBUILD% "%dir%regen.vcxproj" /t:%target% %parallel% %verbose%^
+ /p:IncludeExternals=%IncludeExternals%^
+ /p:Configuration=%conf% /p:Platform=%platf%^
+ /p:UseTestMarker=%UseTestMarker% %GITProperty%^
+ %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+@echo off
+goto :eof
+
 :Version
 rem Display the current build version information
 call "%dir%find_msbuild.bat" %MSBUILD%
diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj
new file mode 100644
index 00000000000..876b12bae9c
--- /dev/null
+++ b/PCbuild/regen.vcxproj
@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|ARM">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|Win32">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|x64">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|ARM">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|Win32">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|x64">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{21CF2108-2CC9-4005-A6ED-B7965ADE3854}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>regen</RootNamespace>
+    <SupportPGO>false</SupportPGO>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="python.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Utility</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="pyproject.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\Programs\_freeze_importlib.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="pythoncore.vcxproj">
+      <Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
+      <Private>true</Private>
+      <ReferenceOutputAssembly>true</ReferenceOutputAssembly>
+      <CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
+      <LinkLibraryDependencies>true</LinkLibraryDependencies>
+      <UseLibraryDependencyInputs>false</UseLibraryDependencyInputs>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\Grammar\Grammar">
+    </None>
+    <None Include="..\Grammar\Tokens">
+    </None>
+    <None Include="..\Include\graminit.h">
+    </None>
+    <None Include="..\Python\graminit.c">
+    </None>
+    <None Include="..\Include\token.h">
+    </None>
+    <None Include="..\Include\opcode.h">
+    </None>
+    <None Include="..\Include\Python-ast.h">
+    </None>
+    <None Include="..\Python\Python-ast.c">
+    </None>
+    <None Include="..\Lib\keyword.py">
+    </None>
+    <None Include="..\Lib\symbol.py">
+    </None>
+    <None Include="..\Doc\library\token-list.inc">
+    </None>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+  <Target Name="_RegenGrammar" BeforeTargets="Build">
+    <!-- Regenerate Include/graminit.h and Python/graminit.c from Grammar/Grammar using pgen-->
+    <Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)graminit.h&quot; &quot;$(IntDir)graminit.c&quot;" />
+    <Copy SourceFiles="$(IntDir)graminit.h" DestinationFiles="$(PySourcePath)Include\graminit.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Copy SourceFiles="$(IntDir)graminit.c" DestinationFiles="$(PySourcePath)Python\graminit.c">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
+    </Copy>
+    <Warning Text="Grammar updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
+    <!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
+    <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
+    <!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
+    <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
+    <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_opcode_h.py &quot;$(PySourcePath)Lib\opcode.py&quot; &quot;$(IntDir)opcode.h&quot;" />
+    <Copy SourceFiles="$(IntDir)opcode.h" DestinationFiles="$(PySourcePath)Include\opcode.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_Updated" />
+    </Copy>
+    <Warning Text="Opcodes updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
+  </Target>
+  <Target Name="_RegenTokens" AfterTargets="_RegenOpcodes">
+    <!-- Regenerate Doc/library/token-list.inc from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py rst &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Doc\library\token-list.inc&quot;" />
+    <!-- Regenerate Include/token.h from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py h &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Include\token.h&quot;" />
+    <!-- Regenerate Parser/token.c from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py c &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Parser\token.c&quot;" />
+    <!-- Regenerate Lib/token.py from Grammar/Tokens using Tools/scripts/generate_token.py -->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py py &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Lib\token.py&quot;" />
+  </Target>
+  <Target Name="_RegenKeywords" AfterTargets="_RegenTokens">
+    <!-- Regenerate Lib/keyword.py from Grammar/Grammar and Grammar/Tokens using Parser/pgen-->
+    <Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen.keywordgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)keyword.py&quot;" />
+    <Copy SourceFiles="$(IntDir)keyword.py" DestinationFiles="$(PySourcePath)Lib\keyword.py">
+      <Output TaskParameter="CopiedFiles" ItemName="_Updated" />
+    </Copy>
+    <Warning Text="Keywords updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
+  </Target>
+  <Target Name="_RegenSymbols" AfterTargets="_RegenKeywords">
+    <!-- Regenerate Lib/symbol.py from Include/graminit.h using Tools/scripts/generate_symbol_py.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_symbol_py.py &quot;$(PySourcePath)Include\graminit.h&quot;  &quot;$(PySourcePath)Lib\symbol.py&quot;" />
+  </Target>
+  <Target Name="_CleanFiles" BeforeTargets="CoreClean">
+    <ItemGroup>
+      <Clean Include="$(IntDir)keyword.py" />
+      <Clean Include="$(IntDir)opcode.h" />
+      <Clean Include="$(IntDir)Python-ast.c" />
+      <Clean Include="$(IntDir)Python-ast.h" />
+      <Clean Include="$(IntDir)graminit.h.new" />
+      <Clean Include="$(IntDir)graminit.c.new" />
+    </ItemGroup>
+  </Target>
+</Project>
\ No newline at end of file