[3.6] bpo-31340: Change to building with MSVC v141 (included with Visual Studio 2017) (GH-3311) (#3386)

.github/appveyor.yml

@@ -14,6 +14,9 @@ test_script:
 - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 environment:
   HOST_PYTHON: C:\Python36\python.exe
+image:
+- Visual Studio 2015
+- Visual Studio 2017
 
 # Only trigger AppVeyor if actual code or its configuration changes
 only_commits:

Doc/make.bat

@@ -6,18 +6,29 @@ pushd %~dp0
 set this=%~n0
 
 call ..\PCBuild\find_python.bat %PYTHON%
-if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
+if not defined SPHINXBUILD if defined PYTHON (
-    set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
+    %PYTHON% -c "import sphinx" > nul 2> nul
-    rem Cannot use %SPHINXBUILD% in the same block where we set it
+    if errorlevel 1 (
-    if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
         echo Installing sphinx with %PYTHON%
-        "%PYTHON%" -m pip install sphinx
+        %PYTHON% -m pip install sphinx
         if errorlevel 1 exit /B
     )
+    set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
 )
 
-if "%PYTHON%" EQU "" set PYTHON=py
+if not defined BLURB if defined PYTHON (
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
+    %PYTHON% -c "import blurb" > nul 2> nul
+    if errorlevel 1 (
+        echo Installing blurb with %PYTHON%
+        %PYTHON% -m pip install blurb
+        if errorlevel 1 exit /B
+    )
+    set BLURB=%PYTHON% -m blurb
+)
+
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
+if not defined BLURB set BLURB=blurb
 
 if "%1" NEQ "htmlhelp" goto :skiphhcsearch
 if exist "%HTMLHELP%" goto :skiphhcsearch
@@ -96,6 +107,19 @@ echo.be passed by setting the SPHINXOPTS environment variable.
 goto end
 
 :build
+if exist ..\Misc\NEWS (
+    echo.Copying Misc\NEWS to build\NEWS
+    copy ..\Misc\NEWS build\NEWS > nul
+) else if exist ..\Misc\NEWS.D (
+    if defined BLURB (
+        echo.Merging Misc/NEWS with %BLURB%
+        %BLURB% merge -f build\NEWS
+    ) else (
+        echo.No Misc/NEWS file and Blurb is not available.
+        exit /B 1
+    )
+)
+
 if NOT "%PAPER%" == "" (
     set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
 )

Lib/distutils/command/bdist_wininst.py

@@ -318,26 +318,30 @@ class bdist_wininst(Command):
         # string compares seem wrong, but are what sysconfig.py itself uses
         if self.target_version and self.target_version < cur_version:
             if self.target_version < "2.4":
-                bv = 6.0
+                bv = '6.0'
             elif self.target_version == "2.4":
-                bv = 7.1
+                bv = '7.1'
             elif self.target_version == "2.5":
-                bv = 8.0
+                bv = '8.0'
             elif self.target_version <= "3.2":
-                bv = 9.0
+                bv = '9.0'
             elif self.target_version <= "3.4":
-                bv = 10.0
+                bv = '10.0'
             else:
-                bv = 14.0
+                bv = '14.0'
         else:
             # for current version - use authoritative check.
             try:
                 from msvcrt import CRT_ASSEMBLY_VERSION
             except ImportError:
                 # cross-building, so assume the latest version
-                bv = 14.0
+                bv = '14.0'
             else:
-                bv = float('.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2]))
+                bv = '.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2])
+                if bv == '14.11':
+                    # v141 and v140 are binary compatible,
+                    # so keep using the 14.0 stub.
+                    bv = '14.0'
 
 
         # wininst-x.y.exe is in the same directory as this file
@@ -353,7 +357,7 @@ class bdist_wininst(Command):
         else:
             sfix = ''
 
-        filename = os.path.join(directory, "wininst-%.1f%s.exe" % (bv, sfix))
+        filename = os.path.join(directory, "wininst-%s%s.exe" % (bv, sfix))
         f = open(filename, "rb")
         try:
             return f.read()

Misc/NEWS.d/next/Windows/2017-09-04-13-19-05.bpo-31340.MbkzLi.rst

@@ -0,0 +1 @@
+Change to building with MSVC v141 (included with Visual Studio 2017)

PCbuild/build.bat

@@ -104,7 +104,7 @@ if "%kill%"=="true" call :Kill
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
-    call :Build
+    call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
     del /s "%dir%\*.pgc"
     del /s "%dir%\..\Lib\*.pyc"
     echo on

PCbuild/find_msbuild.bat

@@ -29,6 +29,14 @@
 @where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
 @if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
 
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
 @rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
 @reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
 @if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
@@ -36,13 +44,6 @@
 )
 @if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
 
-@rem VS 2015 and earlier register MSBuild separately, so we can find it.
-@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
-@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
-    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
-)
-@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
-
 
 @exit /b 1
 

PCbuild/pyproject.props

@@ -147,13 +147,28 @@ foreach (System.Diagnostics.Process p in System.Diagnostics.Process.GetProcesses
              Targets="CleanAll" />
   </Target>
 
+  <Target Name="CopyPGCFiles" BeforeTargets="PrepareForBuild" Condition="$(Configuration) == 'PGUpdate'">
+    <ItemGroup>
+      <_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" />
+      <_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" />
+      <_CopyFiles Include="@(_PGCFiles);@(_PGDFile)" Condition="Exists(%(FullPath))" />
+    </ItemGroup>
+    <Delete Files="@(_CopyFiles->'$(OutDir)%(Filename)%(Extension)')" />
+    <Error Text="PGO run did not succeed (no $(TargetName)!*.pgc files) and there is no data to merge"
+           Condition="$(RequirePGCFiles) == 'true' and @(_PGCFiles) == ''" />
+    <Copy SourceFiles="@(_CopyFiles)"
+          DestinationFolder="$(OutDir)"
+          UseHardLinksIfPossible="true"
+          OverwriteReadOnlyFiles="true" />
+  </Target>
+
   <PropertyGroup>
-    <SdkBinPath Condition="'$(SdkBinPath)' == '' or !Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86</SdkBinPath>
+    <SdkBinPath Condition="'$(SdkBinPath)' == '' or !Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\$(DefaultWindowsSDKVersion)\x86</SdkBinPath>
+    <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86</SdkBinPath>
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86</SdkBinPath>
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86</SdkBinPath>
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\</SdkBinPath>
-    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
+    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>    <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
-    <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
   </PropertyGroup>
   
   <Target Name="_SignBuild" AfterTargets="AfterBuild" Condition="'$(SigningCertificate)' != '' and $(SupportSigning)">

PCbuild/python.props

@@ -10,6 +10,7 @@
 
     We set BasePlatformToolset for ICC's benefit, it's otherwise ignored.
     -->
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath14)' != ''">v140</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath12)' != ''">v120</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath11)' != ''">v110</BasePlatformToolset>
@@ -39,6 +40,7 @@
     <BuildPath Condition="'$(ArchName)' == 'amd64'">$(BuildPath64)</BuildPath>
     <BuildPath Condition="'$(BuildPath)' == ''">$(PySourcePath)PCBuild\$(ArchName)\</BuildPath>
     <BuildPath Condition="!HasTrailingSlash($(BuildPath))">$(BuildPath)\</BuildPath>
+    <BuildPath Condition="$(Configuration) == 'PGInstrument'">$(BuildPath)instrumented\</BuildPath>
     
     <!-- Directories of external projects. tcltk is handled in tcltk.props -->
     <ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
@@ -63,6 +65,23 @@
     <PythonExe Condition="'$(PythonExe)' == ''">$(BuildPath)python$(PyDebugExt).exe</PythonExe>
   </PropertyGroup>
 
+  <PropertyGroup Condition="$(DefaultWindowsSDKVersion) == ''">
+    <!--
+    Attempt to select the latest installed WinSDK. If we don't find any, then we will
+    let the MSBuild targets determine which one it wants to use (typically the earliest
+    possible version). Since we limit WINVER to Windows 7 anyway, it doesn't really
+    matter which WinSDK version we use.
+    -->
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.15063'">10.0.15063.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.15063'">10.0.15063.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.14393'">10.0.14393.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.14393'">10.0.14393.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10586'">10.0.10586.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10586'">10.0.10586.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10240'">10.0.10240.0</DefaultWindowsSDKVersion>
+    <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10240'">10.0.10240.0</DefaultWindowsSDKVersion>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(OverrideVersion)' == ''">
     <!--
     Read version information from Include\patchlevel.h. The following properties are set:

PCbuild/python.vcxproj

@@ -96,6 +96,7 @@ set PYTHONPATH=$(PySourcePath)Lib
     <PropertyGroup>
       <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
       <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
+      <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(VC_PGO_RunTime_Dir) != ''">@set PATH=%PATH%%3B$(VC_PGO_RunTime_Dir)</_PGOPath>
       <_Content>@rem This script invokes the most recently built Python with all arguments
 @rem passed through to the interpreter.  This file is generated by the
 @rem build process and any changes *will* be thrown away by the next

PCbuild/pythoncore.vcxproj

@@ -49,6 +49,7 @@
   </ImportGroup>
   <PropertyGroup>
     <KillPython>true</KillPython>
+    <RequirePGCFiles>true</RequirePGCFiles>
   </PropertyGroup>
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
@@ -428,7 +429,7 @@
       </ClCompile>
     </ItemGroup>
   </Target>
-  <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140'">
+  <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140' and $(PlatformToolset) != 'v141'">
     <Warning Text="Toolset $(PlatformToolset) is not used for official builds. Your build may have errors or incompatibilities." />
   </Target>
 </Project>

Tools/msi/exe/exe.wixproj

@@ -40,5 +40,27 @@
                           Lines="@(_LicenseFiles->'%(Content)')" />
     </Target>
     
+    <Target Name="_CopyMiscNews" AfterTargets="PrepareForBuild" Condition="Exists('$(PySourcePath)Misc\NEWS')">
+        <Copy SourceFiles="$(PySourcePath)Misc\NEWS" DestinationFiles="$(BuildPath)NEWS.txt" />
+    </Target>
+    
+    <Target Name="_MergeMiscNewsWithBlurb" AfterTargets="PrepareForBuild" Condition="$(Blurb) != '' and !Exists('$(PySourcePath)Misc\NEWS')">
+        <Exec Command="$(Blurb) merge -f &quot;$(BuildPath)NEWS.txt&quot;" WorkingDirectory="$(PCBuild)" />
+    </Target>
+    
+    <Target Name="_MergeMiscNewsWithPython" AfterTargets="PrepareForBuild" Condition="$(Blurb) == '' and !Exists('$(PySourcePath)Misc\NEWS')">
+        <ItemGroup>
+            <HostPython Include="$(ExternalsDir)python*\tools\python.exe" />
+            <HostPython Include="@(HostPython)" Condition="Exists(%(FullPath))" />
+            <HostPython Include="py" Condition="@(HostPython) == ''" />
+        </ItemGroup>
+        <PropertyGroup>
+            <HostPython>@(HostPython)</HostPython>
+            <HostPython Condition="$(HostPython.Contains(';'))">$(HostPython.Remove($(HostPython.IndexOf(';'))))</HostPython>
+        </PropertyGroup>
+        <Exec Command="&quot;$(HostPython)&quot; -m pip install -U blurb" WorkingDirectory="$(PCBuild)" />
+        <Exec Command="&quot;$(HostPython)&quot; -m blurb merge -f &quot;$(BuildPath)NEWS.txt&quot;" WorkingDirectory="$(PCBuild)" />
+    </Target>
+    
     <Import Project="..\msi.targets" />
 </Project>

Tools/msi/exe/exe_files.wxs

@@ -6,7 +6,7 @@
                 <File Name="LICENSE.txt" Source="LICENSE" KeyPath="yes" />
             </Component>
             <Component Id="NEWS.txt" Directory="InstallDirectory" Guid="*">
-                <File Name="NEWS.txt" Source="!(bindpath.src)Misc\NEWS" KeyPath="yes" />
+                <File Name="NEWS.txt" KeyPath="yes" />
             </Component>
         </ComponentGroup>
     </Fragment>