diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml index 77458579248..3d072e3b43e 100644 --- a/.azure-pipelines/windows-release.yml +++ b/.azure-pipelines/windows-release.yml @@ -1,7 +1,8 @@ name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) +variables: + __RealSigningCertificate: 'Python Software Foundation' # QUEUE TIME VARIABLES -# variables: # GitRemote: python # SourceTag: # DoPGO: true @@ -13,6 +14,9 @@ name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) # DoEmbed: true # DoMSI: true # DoPublish: false +# PyDotOrgUsername: '' +# PyDotOrgServer: '' +# BuildToPublish: '' trigger: none pr: none @@ -20,18 +24,21 @@ pr: none stages: - stage: Build displayName: Build binaries + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-build.yml - stage: Sign displayName: Sign binaries dependsOn: Build + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-sign.yml - stage: Layout displayName: Generate layouts dependsOn: Sign + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-layout-full.yml - template: windows-release/stage-layout-embed.yml @@ -39,11 +46,13 @@ stages: - stage: Pack dependsOn: Layout + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-pack-nuget.yml - stage: Test dependsOn: Pack + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-test-embed.yml - template: windows-release/stage-test-nuget.yml @@ -51,46 +60,70 @@ stages: - stage: Layout_MSIX displayName: Generate MSIX layouts dependsOn: Sign - condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish']))) jobs: - template: windows-release/stage-layout-msix.yml - stage: Pack_MSIX displayName: Package MSIX dependsOn: Layout_MSIX + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-pack-msix.yml - stage: Build_MSI displayName: Build MSI installer dependsOn: Sign - condition: and(succeeded(), eq(variables['DoMSI'], 'true')) + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish']))) jobs: - template: windows-release/stage-msi.yml - stage: Test_MSI displayName: Test MSI installer dependsOn: Build_MSI + condition: and(succeeded(), not(variables['BuildToPublish'])) jobs: - template: windows-release/stage-test-msi.yml - stage: PublishPyDotOrg displayName: Publish to python.org dependsOn: ['Test_MSI', 'Test'] - condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) jobs: - template: windows-release/stage-publish-pythonorg.yml - stage: PublishNuget displayName: Publish to nuget.org dependsOn: Test - condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) jobs: - template: windows-release/stage-publish-nugetorg.yml - stage: PublishStore displayName: Publish to Store dependsOn: Pack_MSIX - condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-store.yml + + +- stage: PublishExistingPyDotOrg + displayName: Publish existing build to python.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishExistingNuget + displayName: Publish existing build to nuget.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishExistingStore + displayName: Publish existing build to Store + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) jobs: - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/gpg-sign.yml b/.azure-pipelines/windows-release/gpg-sign.yml new file mode 100644 index 00000000000..0855af8d703 --- /dev/null +++ b/.azure-pipelines/windows-release/gpg-sign.yml @@ -0,0 +1,28 @@ +parameters: + GPGKeyFile: $(GPGKey) + GPGPassphrase: $(GPGPassphrase) + Files: '*' + WorkingDirectory: $(Build.BinariesDirectory) + +steps: +- task: DownloadSecureFile@1 + name: gpgkey + inputs: + secureFile: ${{ parameters.GPGKeyFile }} + displayName: 'Download GPG key' + +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" + gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" + (gci -File ${{ parameters.Files }}).FullName | %{ + gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_ + "Made signature for $_" + } + displayName: 'Generate GPG signatures' + workingDirectory: ${{ parameters.WorkingDirectory }} + +- powershell: | + $p = gps "gpg-agent" -EA 0 + if ($p) { $p.Kill() } + displayName: 'Kill GPG agent' + condition: true diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml index 7586d850f34..296eb28648b 100644 --- a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -14,13 +14,26 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: 'Download artifact: nuget' + condition: and(succeeded(), not(variables['BuildToPublish'])) inputs: artifactName: nuget downloadPath: $(Build.BinariesDirectory) + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + - task: NuGetCommand@2 displayName: Push packages - condition: and(succeeded(), eq(variables['SigningCertificate'], 'Python Software Foundation')) + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) inputs: command: push packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml index 2215a56d4bc..2dd354a8c27 100644 --- a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -4,31 +4,151 @@ jobs: condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) pool: - vmName: win2016-vs2017 + #vmName: win2016-vs2017 + name: 'Windows Release' workspace: clean: all steps: - - checkout: none + - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: Doc' + condition: and(succeeded(), not(variables['BuildToPublish'])) inputs: artifactName: Doc - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\Doc - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: msi' + condition: and(succeeded(), not(variables['BuildToPublish'])) inputs: artifactName: msi - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\msi - task: DownloadBuildArtifacts@0 displayName: 'Download artifact: embed' + condition: and(succeeded(), not(variables['BuildToPublish'])) inputs: artifactName: embed downloadPath: $(Build.BinariesDirectory) - # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') - # If we are not real-signed, DO NOT PUBLISH + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): Doc' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): msi' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact from $(BuildToPublish): embed' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + + - template: ./gpg-sign.yml + parameters: + GPGKeyFile: 'python-signing.key' + Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip' + + - powershell: > + $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 + -build msi + -user $(PyDotOrgUsername) + -server $(PyDotOrgServer) + -doc_htmlhelp doc\htmlhelp + -embed embed + -skippurge + -skiptest + -skiphash + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload files to python.org' + + - powershell: > + python + "$(Build.SourcesDirectory)\Tools\msi\purge.py" + (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Purge CDN' + + - powershell: | + $failures = 0 + gci "msi\*\*-webinstall.exe" -File | %{ + $d = mkdir "tests\$($_.BaseName)" -Force + gci $d -r -File | del + $ic = copy $_ $d -PassThru + "Checking layout for $($ic.Name)" + Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" + if (-not $?) { + Write-Error "Failed to validate layout of $($inst.Name)" + $failures += 1 + } + } + if ($failures) { + Write-Error "Failed to validate $failures installers" + exit 1 + } + #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Test layouts' + + - powershell: | + $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | ` + Sort-Object Name | ` + Format-Table Name, @{ + Label="MD5"; + Expression={(Get-FileHash $_ -Algorithm MD5).Hash} + }, Length -AutoSize | ` + Out-String -Width 4096 + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + $hashes | Out-File "$d\hashes.txt" -Encoding ascii + $hashes + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Generate hashes' + + - powershell: | + "Copying:" + (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force + gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Copy GPG signatures for build' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\hashes' + artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml index 06884c4f35b..b22147b1ab4 100644 --- a/.azure-pipelines/windows-release/stage-publish-store.yml +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -14,9 +14,22 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: 'Download artifact: msixupload' + condition: and(succeeded(), not(variables['BuildToPublish'])) inputs: artifactName: msixupload downloadPath: $(Build.BinariesDirectory) - # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) # If we are not real-signed, DO NOT PUBLISH diff --git a/Tools/msi/uploadrelease.ps1 b/Tools/msi/uploadrelease.ps1 index b6fbeea2981..469a96818a1 100644 --- a/Tools/msi/uploadrelease.ps1 +++ b/Tools/msi/uploadrelease.ps1 @@ -164,5 +164,4 @@ if (-not $skiphash) { Out-String -Width 4096 $hashes | clip $hashes - popd }