From 21a92f8cda525d25a165b773fbe1bfffd303a000 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 14 Jun 2019 08:29:20 -0700 Subject: [PATCH] Implement Windows release builds in Azure Pipelines (GH-14065) --- .azure-pipelines/windows-release.yml | 96 +++++++++++ .../windows-release/build-steps.yml | 83 +++++++++ .azure-pipelines/windows-release/checkout.yml | 21 +++ .azure-pipelines/windows-release/find-sdk.yml | 17 ++ .../windows-release/layout-command.yml | 20 +++ .../windows-release/mingw-lib.yml | 13 ++ .../windows-release/msi-steps.yml | 142 ++++++++++++++++ .../windows-release/stage-build.yml | 157 ++++++++++++++++++ .../windows-release/stage-layout-embed.yml | 56 +++++++ .../windows-release/stage-layout-full.yml | 62 +++++++ .../windows-release/stage-layout-msix.yml | 86 ++++++++++ .../windows-release/stage-layout-nuget.yml | 44 +++++ .../windows-release/stage-msi.yml | 36 ++++ .../windows-release/stage-pack-msix.yml | 127 ++++++++++++++ .../windows-release/stage-pack-nuget.yml | 41 +++++ .../stage-publish-nugetorg.yml | 28 ++++ .../stage-publish-pythonorg.yml | 34 ++++ .../windows-release/stage-publish-store.yml | 22 +++ .../windows-release/stage-sign.yml | 113 +++++++++++++ .../windows-release/stage-test-embed.yml | 40 +++++ .../windows-release/stage-test-msi.yml | 108 ++++++++++++ .../windows-release/stage-test-nuget.yml | 58 +++++++ Doc/make.bat | 2 +- {Tools/msi/exe => PC}/crtlicense.txt | 0 PC/layout/main.py | 71 ++++---- PC/layout/support/appxmanifest.py | 62 +++---- PC/layout/support/nuspec.py | 66 ++++++++ PC/layout/support/options.py | 9 +- PC/layout/support/pip.py | 28 +++- PC/layout/support/props.py | 20 +-- PC/python_uwp.cpp | 4 +- PCbuild/_tkinter.vcxproj | 2 +- PCbuild/build.bat | 19 ++- PCbuild/pyproject.props | 18 +- PCbuild/python.props | 4 +- PCbuild/python.vcxproj | 21 ++- PCbuild/tcltk.props | 21 ++- Tools/msi/buildrelease.bat | 2 +- Tools/msi/exe/exe.wixproj | 19 --- Tools/msi/exe/exe_files.wxs | 2 +- Tools/msi/make_cat.ps1 | 8 + Tools/msi/msi.props | 3 +- Tools/msi/msi.targets | 10 +- Tools/msi/sign_build.ps1 | 2 +- Tools/msi/tcltk/tcltk.wixproj | 6 +- Tools/msi/uploadrelease.ps1 | 56 +++++-- 46 files changed, 1692 insertions(+), 167 deletions(-) create mode 100644 .azure-pipelines/windows-release.yml create mode 100644 .azure-pipelines/windows-release/build-steps.yml create mode 100644 .azure-pipelines/windows-release/checkout.yml create mode 100644 .azure-pipelines/windows-release/find-sdk.yml create mode 100644 .azure-pipelines/windows-release/layout-command.yml create mode 100644 .azure-pipelines/windows-release/mingw-lib.yml create mode 100644 .azure-pipelines/windows-release/msi-steps.yml create mode 100644 .azure-pipelines/windows-release/stage-build.yml create mode 100644 .azure-pipelines/windows-release/stage-layout-embed.yml create mode 100644 .azure-pipelines/windows-release/stage-layout-full.yml create mode 100644 .azure-pipelines/windows-release/stage-layout-msix.yml create mode 100644 .azure-pipelines/windows-release/stage-layout-nuget.yml create mode 100644 .azure-pipelines/windows-release/stage-msi.yml create mode 100644 .azure-pipelines/windows-release/stage-pack-msix.yml create mode 100644 .azure-pipelines/windows-release/stage-pack-nuget.yml create mode 100644 .azure-pipelines/windows-release/stage-publish-nugetorg.yml create mode 100644 .azure-pipelines/windows-release/stage-publish-pythonorg.yml create mode 100644 .azure-pipelines/windows-release/stage-publish-store.yml create mode 100644 .azure-pipelines/windows-release/stage-sign.yml create mode 100644 .azure-pipelines/windows-release/stage-test-embed.yml create mode 100644 .azure-pipelines/windows-release/stage-test-msi.yml create mode 100644 .azure-pipelines/windows-release/stage-test-nuget.yml rename {Tools/msi/exe => PC}/crtlicense.txt (100%) create mode 100644 PC/layout/support/nuspec.py diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000..77458579248 --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,96 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +# QUEUE TIME VARIABLES +# variables: +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), eq(variables['DoMSI'], 'true')) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000..508d73b0865 --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,83 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + +- task: PublishBuildArtifacts@1 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' + ArtifactName: bin_$(Name) + +- task: PublishBuildArtifacts@1 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' + ArtifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000..d42d55fff08 --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000..e4de78555b3 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000..3ec9b69ad71 --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,20 @@ +steps: +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "{0}" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{2}" + --temp "{3}" + --include-cat "{2}\python.cat" + --doc-build "{4}"' + -f ( + "$(PYTHON)", + "$(Build.SourcesDirectory)", + (Split-Path -Parent "$(PYTHON)"), + "$(Build.BinariesDirectory)\layout-temp", + "$(Build.BinariesDirectory)\doc" + )) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000..30f7d34fa61 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000..2f80c34eeb7 --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,142 @@ +steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge win32 debug files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin_win32_d + targetFolder: $(Build.BinariesDirectory)\bin_win32 + contents: | + **\*_d.* + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge amd64 debug files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin_amd64_d + targetFolder: $(Build.BinariesDirectory)\bin_amd64 + contents: | + **\*_d.* + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + downloadPath: $(Build.BinariesDirectory) + + - script: | + ren bin_win32 win32 + ren bin_amd64 amd64 + displayName: 'Correct artifact directory names' + workingDirectory: $(Build.BinariesDirectory) + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishBuildArtifacts@1 + displayName: 'Publish MSI' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msi' + ArtifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000..121e4b1a278 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,157 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: doc' + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)\Doc + ArtifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_win32' + ArtifactName: tcltk_lib_win32 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + ArtifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000..c9d58b6b30a --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,56 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' + ArtifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000..3593cf0a3f6 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,62 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' + ArtifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000..1a1e0a2fd68 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,86 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx-store' + ArtifactName: layout_appxstore_$(Name) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + APPX_DATA_PUBLISHER: $(APPX_DATA_PUBLISHER) + APPX_DATA_SHA256: $(APPX_DATA_SHA256) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx' + ArtifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000..ca4213d9e5c --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,44 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\nuget' + ArtifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000..7afc816a0c6 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000..6f1846e581e --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,127 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout_$(Artifact)_$(Name)" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - checkout: none + - template: ./find-sdk.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000..5aa394fa48a --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,41 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout_nuget_$(Name)\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000..7586d850f34 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,28 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], 'Python Software Foundation')) + inputs: + command: push + packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000..2215a56d4bc --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,34 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: Doc' + inputs: + artifactName: Doc + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000..06884c4f35b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,22 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000..3d6ca9457f1 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,113 @@ +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + - template: ./find-sdk.yml + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + continueOnError: true + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_bin_$(Name)' + ArtifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: cert' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\cert' + ArtifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000..ab377fdfa8c --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,40 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + Expand-Archive -Path "$(Build.BinariesDirectory)\embed\embed-$(Name).zip" -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000..10039295a18 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000..1f8b601d0d0 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea9..dfc622f6661 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. diff --git a/Tools/msi/exe/crtlicense.txt b/PC/crtlicense.txt similarity index 100% rename from Tools/msi/exe/crtlicense.txt rename to PC/crtlicense.txt diff --git a/PC/layout/main.py b/PC/layout/main.py index 624033e721b..c39aab208d3 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -31,6 +31,7 @@ from .support.logging import * from .support.options import * from .support.pip import * from .support.props import * +from .support.nuspec import * BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py") BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py" @@ -66,6 +67,7 @@ DATA_DIRS = FileNameSet("data") TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") + def copy_if_modified(src, dest): try: dest_stat = os.stat(dest) @@ -73,12 +75,15 @@ def copy_if_modified(src, dest): do_copy = True else: src_stat = os.stat(src) - do_copy = (src_stat.st_mtime != dest_stat.st_mtime or - src_stat.st_size != dest_stat.st_size) + do_copy = ( + src_stat.st_mtime != dest_stat.st_mtime + or src_stat.st_size != dest_stat.st_size + ) if do_copy: shutil.copy2(src, dest) + def get_lib_layout(ns): def _c(f): if f in EXCLUDE_FROM_LIB: @@ -119,7 +124,7 @@ def get_tcltk_lib(ns): except FileNotFoundError: pass if not tcl_lib or not os.path.isdir(tcl_lib): - warn("Failed to find TCL_LIBRARY") + log_warning("Failed to find TCL_LIBRARY") return for dest, src in rglob(Path(tcl_lib).parent, "**/*"): @@ -168,7 +173,7 @@ def get_layout(ns): for dest, src in rglob(ns.build, "vcruntime*.dll"): yield dest, src - yield "LICENSE.txt", ns.source / "LICENSE" + yield "LICENSE.txt", ns.build / "LICENSE.txt" for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: @@ -222,15 +227,12 @@ def get_layout(ns): yield dest, src if ns.include_pip: - pip_dir = get_pip_dir(ns) - if not pip_dir.is_dir(): - log_warning("Failed to find {} - pip will not be included", pip_dir) - else: - pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}" - for dest, src in rglob(pip_dir, "**/*"): - if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB: - continue - yield pkg_root.format(dest), src + for dest, src in get_pip_layout(ns): + if isinstance(src, tuple) or not ( + src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB + ): + continue + yield dest, src if ns.include_chm: for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): @@ -244,6 +246,10 @@ def get_layout(ns): for dest, src in get_props_layout(ns): yield dest, src + if ns.include_nuspec: + for dest, src in get_nuspec_layout(ns): + yield dest, src + for dest, src in get_appx_layout(ns): yield dest, src @@ -287,7 +293,9 @@ def _py_temp_compile(src, ns, dest_dir=None, checked=True): return None dest = (dest_dir or ns.temp) / (src.stem + ".py") - return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked) + return _compile_one_py( + src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked + ) def _write_to_zip(zf, dest, src, ns, checked=True): @@ -361,28 +369,9 @@ def generate_source_files(ns): print("# Uncomment to run site.main() automatically", file=f) print("#import site", file=f) - if ns.include_appxmanifest: - log_info("Generating AppxManifest.xml in {}", ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - - with open(ns.temp / "AppxManifest.xml", "wb") as f: - f.write(get_appxmanifest(ns)) - - with open(ns.temp / "_resources.xml", "wb") as f: - f.write(get_resources_xml(ns)) - if ns.include_pip: - pip_dir = get_pip_dir(ns) - if not (pip_dir / "pip").is_dir(): - log_info("Extracting pip to {}", pip_dir) - pip_dir.mkdir(parents=True, exist_ok=True) - extract_pip_files(ns) - - if ns.include_props: - log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f: - f.write(get_props(ns)) + log_info("Extracting pip") + extract_pip_files(ns) def _create_zip_file(ns): @@ -427,6 +416,18 @@ def copy_files(files, ns): log_info("Processed {} files", count) log_debug("Processing {!s}", src) + if isinstance(src, tuple): + src, content = src + if ns.copy: + log_debug("Copy {} -> {}", src, ns.copy / dest) + (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) + with open(ns.copy / dest, "wb") as f: + f.write(content) + if ns.zip: + log_debug("Zip {} into {}", src, ns.zip) + zip_file.writestr(str(dest), content) + continue + if ( ns.precompile and src in PY_FILES diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index 49a35fa1f04..58fba8443f1 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -17,12 +17,7 @@ from xml.etree import ElementTree as ET from .constants import * -__all__ = [] - - -def public(f): - __all__.append(f.__name__) - return f +__all__ = ["get_appx_layout"] APPX_DATA = dict( @@ -166,9 +161,7 @@ REGISTRY = { "Help": { "Main Python Documentation": { "_condition": lambda ns: ns.include_chm, - "": "[{{AppVPackageRoot}}]\\Doc\\{}".format( - PYTHON_CHM_NAME - ), + "": "[{{AppVPackageRoot}}]\\Doc\\{}".format(PYTHON_CHM_NAME), }, "Local Python Documentation": { "_condition": lambda ns: ns.include_html_doc, @@ -239,31 +232,6 @@ def _fixup_sccd(ns, sccd, new_hash=None): return sccd -@public -def get_appx_layout(ns): - if not ns.include_appxmanifest: - return - - yield "AppxManifest.xml", ns.temp / "AppxManifest.xml" - yield "_resources.xml", ns.temp / "_resources.xml" - icons = ns.source / "PC" / "icons" - yield "_resources/pythonx44.png", icons / "pythonx44.png" - yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png" - yield "_resources/pythonx50.png", icons / "pythonx50.png" - yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png" - yield "_resources/pythonx150.png", icons / "pythonx150.png" - yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png" - yield "_resources/pythonwx44.png", icons / "pythonwx44.png" - yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png" - yield "_resources/pythonwx150.png", icons / "pythonwx150.png" - yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png" - sccd = ns.source / SCCD_FILENAME - if sccd.is_file(): - # This should only be set for side-loading purposes. - sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256")) - yield sccd.name, sccd - - def find_or_add(xml, element, attr=None, always_add=False): if always_add: e = None @@ -393,7 +361,6 @@ def disable_registry_virtualization(xml): e = find_or_add(e, "rescap:Capability", ("Name", "unvirtualizedResources")) -@public def get_appxmanifest(ns): for k, v in APPXMANIFEST_NS.items(): ET.register_namespace(k, v) @@ -481,6 +448,29 @@ def get_appxmanifest(ns): return buffer.getbuffer() -@public def get_resources_xml(ns): return RESOURCES_XML_TEMPLATE.encode("utf-8") + + +def get_appx_layout(ns): + if not ns.include_appxmanifest: + return + + yield "AppxManifest.xml", ("AppxManifest.xml", get_appxmanifest(ns)) + yield "_resources.xml", ("_resources.xml", get_resources_xml(ns)) + icons = ns.source / "PC" / "icons" + yield "_resources/pythonx44.png", icons / "pythonx44.png" + yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png" + yield "_resources/pythonx50.png", icons / "pythonx50.png" + yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png" + yield "_resources/pythonx150.png", icons / "pythonx150.png" + yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png" + yield "_resources/pythonwx44.png", icons / "pythonwx44.png" + yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png" + yield "_resources/pythonwx150.png", icons / "pythonwx150.png" + yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png" + sccd = ns.source / SCCD_FILENAME + if sccd.is_file(): + # This should only be set for side-loading purposes. + sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256")) + yield sccd.name, sccd diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py new file mode 100644 index 00000000000..ba26ff337e9 --- /dev/null +++ b/PC/layout/support/nuspec.py @@ -0,0 +1,66 @@ +""" +Provides .props file. +""" + +import os + +from .constants import * + +__all__ = ["get_nuspec_layout"] + +PYTHON_NUSPEC_NAME = "python.nuspec" + +NUSPEC_DATA = { + "PYTHON_TAG": VER_DOT, + "PYTHON_VERSION": os.getenv("PYTHON_NUSPEC_VERSION"), + "PYTHON_BITNESS": "64-bit" if IS_X64 else "32-bit", + "PACKAGENAME": os.getenv("PYTHON_NUSPEC_PACKAGENAME"), + "PACKAGETITLE": os.getenv("PYTHON_NUSPEC_PACKAGETITLE"), + "FILELIST": r' ', +} + +if not NUSPEC_DATA["PYTHON_VERSION"]: + if VER_NAME: + NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}-{}{}".format( + VER_DOT, VER_MICRO, VER_NAME, VER_SERIAL + ) + else: + NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}".format(VER_DOT, VER_MICRO) + +if not NUSPEC_DATA["PACKAGETITLE"]: + NUSPEC_DATA["PACKAGETITLE"] = "Python" if IS_X64 else "Python (32-bit)" + +if not NUSPEC_DATA["PACKAGENAME"]: + NUSPEC_DATA["PACKAGENAME"] = "python" if IS_X64 else "pythonx86" + +FILELIST_WITH_PROPS = r""" + """ + +NUSPEC_TEMPLATE = r""" + + + {PACKAGENAME} + {PACKAGETITLE} + {PYTHON_VERSION} + Python Software Foundation + tools\LICENSE.txt + https://www.python.org/ + Installs {PYTHON_BITNESS} Python for use in build scenarios. + https://www.python.org/static/favicon.ico + python + + +{FILELIST} + + +""" + + +def get_nuspec_layout(ns): + if ns.include_all or ns.include_nuspec: + data = NUSPEC_DATA + if ns.include_all or ns.include_props: + data = dict(data) + data["FILELIST"] = FILELIST_WITH_PROPS + nuspec = NUSPEC_TEMPLATE.format_map(data) + yield "python.nuspec", ("python.nuspec", nuspec.encode("utf-8")) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 00f05667ebb..c8ae4e30a8c 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -30,6 +30,7 @@ OPTIONS = { "launchers": {"help": "specific launchers"}, "appxmanifest": {"help": "an appxmanifest"}, "props": {"help": "a python.props file"}, + "nuspec": {"help": "a python.nuspec file"}, "chm": {"help": "the CHM documentation"}, "html-doc": {"help": "the HTML documentation"}, } @@ -60,13 +61,11 @@ PRESETS = { "stable", "distutils", "venv", - "props" + "props", + "nuspec", ], }, - "iot": { - "help": "Windows IoT Core", - "options": ["stable", "pip"], - }, + "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]}, "default": { "help": "development kit package", "options": [ diff --git a/PC/layout/support/pip.py b/PC/layout/support/pip.py index 369a923ce13..eada456655e 100644 --- a/PC/layout/support/pip.py +++ b/PC/layout/support/pip.py @@ -11,15 +11,11 @@ import shutil import subprocess import sys -__all__ = [] +from .filesets import * + +__all__ = ["extract_pip_files", "get_pip_layout"] -def public(f): - __all__.append(f.__name__) - return f - - -@public def get_pip_dir(ns): if ns.copy: if ns.zip_lib: @@ -29,10 +25,23 @@ def get_pip_dir(ns): return ns.temp / "packages" -@public +def get_pip_layout(ns): + pip_dir = get_pip_dir(ns) + if not pip_dir.is_dir(): + log_warning("Failed to find {} - pip will not be included", pip_dir) + else: + pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}" + for dest, src in rglob(pip_dir, "**/*"): + yield pkg_root.format(dest), src + yield "pip.ini", ("pip.ini", b"[global]\nuser=yes") + + def extract_pip_files(ns): dest = get_pip_dir(ns) - dest.mkdir(parents=True, exist_ok=True) + try: + dest.mkdir(parents=True, exist_ok=False) + except IOError: + return src = ns.source / "Lib" / "ensurepip" / "_bundled" @@ -58,6 +67,7 @@ def extract_pip_files(ns): "--target", str(dest), "--no-index", + "--no-compile", "--no-cache-dir", "-f", str(src), diff --git a/PC/layout/support/props.py b/PC/layout/support/props.py index 3a047d21505..4d3b06195f6 100644 --- a/PC/layout/support/props.py +++ b/PC/layout/support/props.py @@ -6,13 +6,7 @@ import os from .constants import * -__all__ = ["PYTHON_PROPS_NAME"] - - -def public(f): - __all__.append(f.__name__) - return f - +__all__ = ["get_props_layout"] PYTHON_PROPS_NAME = "python.props" @@ -97,14 +91,8 @@ PROPS_TEMPLATE = r""" """ -@public def get_props_layout(ns): if ns.include_all or ns.include_props: - yield "python.props", ns.temp / "python.props" - - -@public -def get_props(ns): - # TODO: Filter contents of props file according to included/excluded items - props = PROPS_TEMPLATE.format_map(PROPS_DATA) - return props.encode("utf-8") + # TODO: Filter contents of props file according to included/excluded items + props = PROPS_TEMPLATE.format_map(PROPS_DATA) + yield "python.props", ("python.props", props.encode("utf-8")) diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 5c8caa6666c..dd1edde7309 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -182,9 +182,9 @@ wmain(int argc, wchar_t **argv) if (*p++ == L'\\') { if (wcsnicmp(p, L"pip", 3) == 0) { moduleName = L"pip"; + /* No longer required when pip 19.1 is added */ _wputenv_s(L"PIP_USER", L"true"); - } - else if (wcsnicmp(p, L"idle", 4) == 0) { + } else if (wcsnicmp(p, L"idle", 4) == 0) { moduleName = L"idlelib"; } } diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index fdfa59648aa..af813b77c1d 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -122,7 +122,7 @@ - + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6f0c85e4a45..bce599329e7 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -76,7 +76,7 @@ if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts -if "%~1"=="-V" shift & goto Version +if "%~1"=="-V" shift & goto :Version rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -111,10 +111,16 @@ call "%dir%find_msbuild.bat" %MSBUILD% if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if "%kill%"=="true" call :Kill +if ERRORLEVEL 1 exit /B 3 if "%do_pgo%"=="true" ( set conf=PGInstrument call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9 +) +rem %VARS% are evaluated eagerly, which would lose the ERRORLEVEL +rem value if we didn't split it out here. +if "%do_pgo%"=="true" if ERRORLEVEL 1 exit /B %ERRORLEVEL% +if "%do_pgo%"=="true" ( del /s "%dir%\*.pgc" del /s "%dir%\..\Lib\*.pyc" echo on @@ -124,7 +130,8 @@ if "%do_pgo%"=="true" ( set conf=PGUpdate set target=Build ) -goto Build +goto :Build + :Kill echo on %MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ @@ -132,7 +139,7 @@ echo on /p:KillPython=true @echo off -goto :eof +exit /B %ERRORLEVEL% :Build rem Call on MSBuild to do the work, echo the command. @@ -148,9 +155,11 @@ echo on %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off -goto :eof +exit /b %ERRORLEVEL% :Version rem Display the current build version information call "%dir%find_msbuild.bat" %MSBUILD% -if not ERRORLEVEL 1 %MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) +%MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +if ERRORLEVEL 1 exit /b 3 \ No newline at end of file diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 12f07dd5128..7c0f50be9ea 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -1,6 +1,8 @@ - - + + + + <__PyProject_Props_Imported>true <_ProjectFileVersion>10.0.30319.1 10.0 $(BuildPath) @@ -29,7 +31,7 @@ $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - + MaxSpeed true true @@ -147,15 +149,15 @@ public override bool Execute() { - + - + @@ -189,8 +191,8 @@ public override bool Execute() { $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" diff --git a/PCbuild/python.props b/PCbuild/python.props index e6642fc4818..b13837d394b 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -1,6 +1,7 @@ - + + <__Python_Props_Imported>true Win32 Release + + + <_TclTkLib Include="$(tcltkdir)\lib\**\*" /> + + + diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 45e189b537f..b72eedecb23 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -29,7 +29,7 @@ set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename} set D=%~dp0 set PCBUILD=%D%..\..\PCbuild\ -if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD% +if NOT DEFINED Py_OutDir set Py_OutDir=%PCBUILD% set EXTERNALS=%D%..\..\externals\windows-installer\ set BUILDX86= diff --git a/Tools/msi/exe/exe.wixproj b/Tools/msi/exe/exe.wixproj index 071501ce6e6..326766bf2d4 100644 --- a/Tools/msi/exe/exe.wixproj +++ b/Tools/msi/exe/exe.wixproj @@ -21,25 +21,6 @@ - - - - <_LicenseFiles Include="@(LicenseFiles)"> - $([System.IO.File]::ReadAllText(%(FullPath))) - - - - - - diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs index 394b4de4735..483d06c65b2 100644 --- a/Tools/msi/exe/exe_files.wxs +++ b/Tools/msi/exe/exe_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/msi/make_cat.ps1 b/Tools/msi/make_cat.ps1 index cc3cd4a2b50..9ea3ddd4957 100644 --- a/Tools/msi/make_cat.ps1 +++ b/Tools/msi/make_cat.ps1 @@ -7,6 +7,8 @@ The path to the catalog definition file to compile and sign. It is assumed that the .cat file will be the same name with a new extension. +.Parameter outfile + The path to move the built .cat file to (optional). .Parameter description The description to add to the signature (optional). .Parameter certname @@ -16,6 +18,7 @@ #> param( [Parameter(Mandatory=$true)][string]$catalog, + [string]$outfile, [switch]$sign, [string]$description, [string]$certname, @@ -35,3 +38,8 @@ if (-not $?) { if ($sign) { Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files @($catalog -replace 'cdf$', 'cat') } + +if ($outfile) { + Split-Path -Parent $outfile | ?{ $_ } | %{ mkdir -Force $_; } + Move-Item ($catalog -replace 'cdf$', 'cat') $outfile +} diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 5da901c0215..3f14501446a 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -56,6 +56,7 @@ true $(ExternalsDir)\windows-installer\redist-1\$(Platform) $([System.IO.Path]::GetFullPath($(CRTRedist))) + $(tcltkDir)lib python$(MajorVersionNumber)$(MinorVersionNumber)$(MicroVersionNumber)$(ReleaseLevelName).chm $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 @@ -121,7 +122,7 @@ src - + tcltk diff --git a/Tools/msi/msi.targets b/Tools/msi/msi.targets index 9283a1ed6c3..4788a637a5d 100644 --- a/Tools/msi/msi.targets +++ b/Tools/msi/msi.targets @@ -47,7 +47,7 @@ EncodingType= - @@ -76,18 +76,18 @@ EncodingType= - + - + - + - + \ No newline at end of file diff --git a/Tools/msi/sign_build.ps1 b/Tools/msi/sign_build.ps1 index 6668eb33a2d..d3f750454f5 100644 --- a/Tools/msi/sign_build.ps1 +++ b/Tools/msi/sign_build.ps1 @@ -16,7 +16,7 @@ #> param( [Parameter(Mandatory=$true)][string]$root, - [string[]]$patterns=@("*.exe", "*.dll", "*.pyd"), + [string[]]$patterns=@("*.exe", "*.dll", "*.pyd", "*.cat"), [string]$description, [string]$certname, [string]$certsha1, diff --git a/Tools/msi/tcltk/tcltk.wixproj b/Tools/msi/tcltk/tcltk.wixproj index fae353f5f50..218f3d15ec8 100644 --- a/Tools/msi/tcltk/tcltk.wixproj +++ b/Tools/msi/tcltk/tcltk.wixproj @@ -20,10 +20,10 @@ - - $(tcltkDir) + + $(TclTkLibraryDir) !(bindpath.tcltk) - $(tcltkDir)lib + $(TclTkLibraryDir) tcl\ tcltk_lib diff --git a/Tools/msi/uploadrelease.ps1 b/Tools/msi/uploadrelease.ps1 index 491df80be1e..b6fbeea2981 100644 --- a/Tools/msi/uploadrelease.ps1 +++ b/Tools/msi/uploadrelease.ps1 @@ -15,6 +15,10 @@ The subdirectory on the host to copy files to. .Parameter tests The path to run download tests in. +.Parameter doc_htmlhelp + Optional path besides -build to locate CHM files. +.Parameter embed + Optional path besides -build to locate ZIP files. .Parameter skipupload Skip uploading .Parameter skippurge @@ -30,6 +34,8 @@ param( [string]$server="python-downloads", [string]$target="/srv/www.python.org/ftp/python", [string]$tests=${env:TEMP}, + [string]$doc_htmlhelp=$null, + [string]$embed=$null, [switch]$skipupload, [switch]$skippurge, [switch]$skiptest, @@ -73,32 +79,45 @@ if (-not $skipupload) { "Upload using $pscp and $plink" "" - pushd $build - $doc = gci python*.chm, python*.chm.asc + if ($doc_htmlhelp) { + pushd $doc_htmlhelp + } else { + pushd $build + } + $chm = gci python*.chm, python*.chm.asc popd $d = "$target/$($p[0])/" & $plink -batch $user@$server mkdir $d & $plink -batch $user@$server chgrp downloads $d & $plink -batch $user@$server chmod g-x,o+rx $d - & $pscp -batch $doc.FullName "$user@${server}:$d" + & $pscp -batch $chm.FullName "$user@${server}:$d" - foreach ($a in gci "$build" -Directory) { + $dirs = gci "$build" -Directory + if ($embed) { + $dirs = ($dirs, (gi $embed)) | %{ $_ } + } + + foreach ($a in $dirs) { "Uploading files from $($a.FullName)" pushd "$($a.FullName)" $exe = gci *.exe, *.exe.asc, *.zip, *.zip.asc $msi = gci *.msi, *.msi.asc, *.msu, *.msu.asc popd - & $pscp -batch $exe.FullName "$user@${server}:$d" + if ($exe) { + & $pscp -batch $exe.FullName "$user@${server}:$d" + } - $sd = "$d$($a.Name)$($p[1])/" - & $plink -batch $user@$server mkdir $sd - & $plink -batch $user@$server chgrp downloads $sd - & $plink -batch $user@$server chmod g-x,o+rx $sd - & $pscp -batch $msi.FullName "$user@${server}:$sd" - & $plink -batch $user@$server chgrp downloads $sd* - & $plink -batch $user@$server chmod g-x,o+r $sd* + if ($msi) { + $sd = "$d$($a.Name)$($p[1])/" + & $plink -batch $user@$server mkdir $sd + & $plink -batch $user@$server chgrp downloads $sd + & $plink -batch $user@$server chmod g-x,o+rx $sd + & $pscp -batch $msi.FullName "$user@${server}:$sd" + & $plink -batch $user@$server chgrp downloads $sd* + & $plink -batch $user@$server chmod g-x,o+r $sd* + } } & $plink -batch $user@$server chgrp downloads $d* @@ -128,7 +147,18 @@ if (-not $skiptest) { if (-not $skiphash) { # Display MD5 hash and size of each downloadable file pushd $build - $hashes = gci python*.chm, *\*.exe, *\*.zip | ` + $files = gci python*.chm, *\*.exe, *\*.zip + if ($doc_htmlhelp) { + cd $doc_htmlhelp + $files = ($files, (gci python*.chm)) | %{ $_ } + } + if ($embed) { + cd $embed + $files = ($files, (gci *.zip)) | %{ $_ } + } + popd + + $hashes = $files | ` Sort-Object Name | ` Format-Table Name, @{Label="MD5"; Expression={(Get-FileHash $_ -Algorithm MD5).Hash}}, Length -AutoSize | ` Out-String -Width 4096