From 57675090b0fe7d6c7d72e56384dc2ff6798f1723 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 24 Sep 2018 07:44:50 -0400 Subject: [PATCH] bpo-34582: Update syntax of Azure Pipelines builds (GH-9521) --- .azure-pipelines/ci.yml | 136 ++++++++++++++++++ .azure-pipelines/docker-steps.yml | 76 ++++++++++ .azure-pipelines/docs-steps.yml | 46 ++++++ .../macos-steps.yml | 27 +--- .../posix-deps.sh | 9 +- .azure-pipelines/posix-steps.yml | 63 ++++++++ .azure-pipelines/pr.yml | 86 +++++++++++ .azure-pipelines/prebuild-checks.yml | 36 +++++ .azure-pipelines/windows-steps.yml | 32 +++++ .vsts/docs-release.yml | 43 ------ .vsts/docs.yml | 58 -------- .vsts/linux-buildbot.yml | 59 -------- .vsts/linux-coverage.yml | 75 ---------- .vsts/linux-pr.yml | 84 ----------- .vsts/macos-pr.yml | 64 --------- .vsts/windows-buildbot.yml | 58 -------- .vsts/windows-pr.yml | 70 --------- Lib/test/test_zlib.py | 23 ++- 18 files changed, 503 insertions(+), 542 deletions(-) create mode 100644 .azure-pipelines/ci.yml create mode 100644 .azure-pipelines/docker-steps.yml create mode 100644 .azure-pipelines/docs-steps.yml rename .vsts/macos-buildbot.yml => .azure-pipelines/macos-steps.yml (57%) rename .vsts/install_deps.sh => .azure-pipelines/posix-deps.sh (51%) create mode 100644 .azure-pipelines/posix-steps.yml create mode 100644 .azure-pipelines/pr.yml create mode 100644 .azure-pipelines/prebuild-checks.yml create mode 100644 .azure-pipelines/windows-steps.yml delete mode 100644 .vsts/docs-release.yml delete mode 100644 .vsts/docs.yml delete mode 100644 .vsts/linux-buildbot.yml delete mode 100644 .vsts/linux-coverage.yml delete mode 100644 .vsts/linux-pr.yml delete mode 100644 .vsts/macos-pr.yml delete mode 100644 .vsts/windows-buildbot.yml delete mode 100644 .vsts/windows-pr.yml diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml new file mode 100644 index 00000000000..f56254e3802 --- /dev/null +++ b/.azure-pipelines/ci.yml @@ -0,0 +1,136 @@ +variables: + manylinux: false + coverage: false + +jobs: +- job: Prebuild + displayName: Pre-build checks + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./prebuild-checks.yml + + +- job: Docs_PR + displayName: Docs PR + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./docs-steps.yml + parameters: + upload: true + + +- job: macOS_CI_Tests + displayName: macOS CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + variables: + testRunTitle: '$(build.sourceBranchName)-macos' + testRunPlatform: macos + + pool: + vmImage: xcode9-macos10.13 + + steps: + - template: ./macos-steps.yml + + +- job: Ubuntu_CI_Tests + displayName: Ubuntu CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(build.sourceBranchName)-linux' + testRunPlatform: linux + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + + +- job: ManyLinux1_CI_Tests + displayName: ManyLinux1 CI Tests + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['manylinux'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(build.sourceBranchName)-manylinux1' + testRunPlatform: manylinux1 + imageName: 'dockcross/manylinux-x64' + + steps: + - template: ./docker-steps.yml + + +- job: Ubuntu_Coverage_CI_Tests + displayName: Ubuntu CI Tests (coverage) + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['coverage'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(Build.SourceBranchName)-linux-coverage' + testRunPlatform: linux-coverage + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + parameters: + coverage: true + + +- job: Windows_CI_Tests + displayName: Windows CI Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: vs2017-win2017 + + strategy: + matrix: + win32: + arch: win32 + buildOpt: + testRunTitle: '$(Build.SourceBranchName)-win32' + testRunPlatform: win32 + win64: + arch: amd64 + buildOpt: '-p x64' + testRunTitle: '$(Build.SourceBranchName)-win64' + testRunPlatform: win64 + maxParallel: 2 + + steps: + - template: ./windows-steps.yml diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml new file mode 100644 index 00000000000..ba4dfd72dd8 --- /dev/null +++ b/.azure-pipelines/docker-steps.yml @@ -0,0 +1,76 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- ${{ if ne(parameters.targetBranch, '') }}: + - script: | + git fetch -q origin ${{ parameters.targetbranch }} + if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated, stopping build process." + echo "##vso[task.setvariable variable=DocOnly]true" + exit + fi + displayName: Detect doc-only changes + +- task: docker@0 + displayName: 'Configure CPython (debug)' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: './configure --with-pydebug' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Build CPython' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make -s -j4' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Display build info' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make pythoninfo' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: docker@0 + displayName: 'Tests' + inputs: + action: 'Run an image' + imageName: $(imageName) + volumes: | + $(build.sourcesDirectory):/src + $(build.binariesDirectory):/build + workDir: '/src' + containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"' + detached: false + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(build.binariesDirectory)/test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml new file mode 100644 index 00000000000..c0404aebdcc --- /dev/null +++ b/.azure-pipelines/docs-steps.yml @@ -0,0 +1,46 @@ +parameters: + latex: false + upload: false + +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + +- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme + displayName: 'Install build dependencies' + +- ${{ if ne(parameters.latex, 'true') }}: + - script: make check suspicious html PYTHON=python + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + +- ${{ if eq(parameters.latex, 'true') }}: + - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full + displayName: 'Install LaTeX' + + - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + +- ${{ if eq(parameters.upload, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish docs' + + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/build' + ArtifactName: docs + publishLocation: Container + + - ${{ if eq(parameters.latex, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish dist' + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/dist' + ArtifactName: docs_dist + publishLocation: Container diff --git a/.vsts/macos-buildbot.yml b/.azure-pipelines/macos-steps.yml similarity index 57% rename from .vsts/macos-buildbot.yml rename to .azure-pipelines/macos-steps.yml index d9c190c4613..64708168945 100644 --- a/.vsts/macos-buildbot.yml +++ b/.azure-pipelines/macos-steps.yml @@ -1,30 +1,9 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: Hosted macOS - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -#variables: - steps: - checkout: self clean: true fetchDepth: 5 -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts +- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev displayName: 'Configure CPython (debug)' - script: make -s -j4 @@ -41,6 +20,6 @@ steps: inputs: testResultsFiles: '$(build.binariesDirectory)/test-results.xml' mergeTestResults: true - testRunTitle: '$(build.sourceBranchName)-macOS' - platform: macOS + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) condition: succeededOrFailed() diff --git a/.vsts/install_deps.sh b/.azure-pipelines/posix-deps.sh similarity index 51% rename from .vsts/install_deps.sh rename to .azure-pipelines/posix-deps.sh index b1fa576047e..a5721075660 100755 --- a/.vsts/install_deps.sh +++ b/.azure-pipelines/posix-deps.sh @@ -1,4 +1,4 @@ -sudo apt-get update || true +sudo apt-get update sudo apt-get -yq install \ build-essential \ @@ -17,3 +17,10 @@ sudo apt-get -yq install \ libffi-dev \ uuid-dev \ xvfb + +if [ ! -z "$1" ] +then + echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 + echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 + python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux +fi diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml new file mode 100644 index 00000000000..429381a7e24 --- /dev/null +++ b/.azure-pipelines/posix-steps.yml @@ -0,0 +1,63 @@ +parameters: + coverage: false + +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- script: ./.azure-pipelines/posix-deps.sh $(openssl_version) + displayName: 'Install dependencies' + +- script: ./configure --with-pydebug + displayName: 'Configure CPython (debug)' + +- script: make -s -j4 + displayName: 'Build CPython' + +- ${{ if eq(parameters.coverage, 'true') }}: + - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage + displayName: 'Set up virtual environment' + + - script: ./venv/bin/python -m test.pythoninfo + displayName: 'Display build info' + + - script: | + xvfb-run ./venv/bin/python -m coverage run --pylib -m test \ + --fail-env-changed \ + -uall,-cpu \ + --junit-xml=$(build.binariesDirectory)/test-results.xml" \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures + displayName: 'Tests with coverage' + + - script: ./venv/bin/python -m coverage xml + displayName: 'Generate coverage.xml' + + - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) + displayName: 'Publish code coverage results' + + +- ${{ if ne(parameters.coverage, 'true') }}: + - script: make pythoninfo + displayName: 'Display build info' + + - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" + displayName: 'Tests' + + +- script: python Tools/scripts/patchcheck.py --travis true + displayName: 'Run patchcheck.py' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(build.binariesDirectory)/test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml new file mode 100644 index 00000000000..653f55b69b1 --- /dev/null +++ b/.azure-pipelines/pr.yml @@ -0,0 +1,86 @@ +jobs: +- job: Prebuild + displayName: Pre-build checks + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./prebuild-checks.yml + + +- job: Docs_PR + displayName: Docs PR + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + steps: + - template: ./docs-steps.yml + + +- job: macOS_PR_Tests + displayName: macOS PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + variables: + testRunTitle: '$(system.pullRequest.TargetBranch)-macos' + testRunPlatform: macos + + pool: + vmImage: xcode9-macos10.13 + + steps: + - template: ./macos-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) + + +- job: Ubuntu_PR_Tests + displayName: Ubuntu PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(system.pullRequest.TargetBranch)-linux' + testRunPlatform: linux + openssl_version: 1.1.0g + + steps: + - template: ./posix-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) + + +- job: Windows_PR_Tests + displayName: Windows PR Tests + dependsOn: Prebuild + condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + + pool: + vmImage: vs2017-win2017 + + strategy: + matrix: + win32: + arch: win32 + buildOpt: + testRunTitle: '$(System.PullRequest.TargetBranch)-win32' + testRunPlatform: win32 + win64: + arch: amd64 + buildOpt: '-p x64' + testRunTitle: '$(System.PullRequest.TargetBranch)-win64' + testRunPlatform: win64 + maxParallel: 2 + + steps: + - template: ./windows-steps.yml + parameters: + targetBranch: $(System.PullRequest.TargetBranch) diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml new file mode 100644 index 00000000000..30ff642d126 --- /dev/null +++ b/.azure-pipelines/prebuild-checks.yml @@ -0,0 +1,36 @@ +steps: +- checkout: self + fetchDepth: 5 + +- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1" + displayName: Set default diff target + +- script: | + git fetch -q origin $(System.PullRequest.TargetBranch) + echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)" + displayName: Fetch comparison tree + condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) + +- script: | + if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)' + then + echo "No docs were updated: docs.run=false" + echo "##vso[task.setvariable variable=run;isOutput=true]false" + else + echo "Docs were updated: docs.run=true" + echo "##vso[task.setvariable variable=run;isOutput=true]true" + fi + displayName: Detect documentation changes + name: docs + +- script: | + if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated: tests.run=false" + echo "##vso[task.setvariable variable=run;isOutput=true]false" + else + echo "Code was updated: tests.run=true" + echo "##vso[task.setvariable variable=run;isOutput=true]true" + fi + displayName: Detect source changes + name: tests diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml new file mode 100644 index 00000000000..d8d5f1753a0 --- /dev/null +++ b/.azure-pipelines/windows-steps.yml @@ -0,0 +1,32 @@ +steps: +- checkout: self + clean: true + fetchDepth: 5 + +- powershell: | + # Relocate build outputs outside of source directory to make cleaning faster + Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' + # UNDONE: Do not build to a different directory because of broken tests + Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' + Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals' + displayName: Update build locations + +- script: PCbuild\build.bat -e $(buildOpt) + displayName: 'Build CPython' + +- script: python.bat -m test.pythoninfo + displayName: 'Display build info' + +- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" + displayName: 'Tests' + env: + PREFIX: $(Py_OutDir)\$(arch) + +- task: PublishTestResults@2 + displayName: 'Publish Test Results' + inputs: + testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' + mergeTestResults: true + testRunTitle: $(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff --git a/.vsts/docs-release.yml b/.vsts/docs-release.yml deleted file mode 100644 index e90428a4249..00000000000 --- a/.vsts/docs-release.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: Hosted Linux Preview - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full - displayName: 'Install LaTeX' - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - -- script: python -m pip install sphinx blurb python-docs-theme - displayName: 'Install build dependencies' - -- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- task: PublishBuildArtifacts@1 - displayName: 'Publish build' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: build - publishLocation: Container - -- task: PublishBuildArtifacts@1 - displayName: 'Publish dist' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/dist' - ArtifactName: dist - publishLocation: Container diff --git a/.vsts/docs.yml b/.vsts/docs.yml deleted file mode 100644 index 0be07b31dfc..00000000000 --- a/.vsts/docs.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: 'Hosted Ubuntu 1604' - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - include: - - Doc/* - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qE '(\.rst$|^Doc|^Misc)' - then - echo "No docs were updated, stopping build process." - echo "##vso[task.setvariable variable=NoDocs]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme - displayName: 'Install build dependencies' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- script: make check suspicious html PYTHON=python - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- task: PublishBuildArtifacts@1 - displayName: 'Publish build' - condition: and(and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')), ne(variables['NoDocs'], 'true')) - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: build - publishLocation: Container diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml deleted file mode 100644 index 417125ae3a4..00000000000 --- a/.vsts/linux-buildbot.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: 'Hosted Ubuntu 1604' - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -#- template: linux-deps.yml - -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' -- script: ./.vsts/install_deps.sh - displayName: 'Install dependencies' -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -s -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: '$(build.sourceBranchName)-linux' - platform: linux - condition: succeededOrFailed() diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml deleted file mode 100644 index cc03e4258ab..00000000000 --- a/.vsts/linux-coverage.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: 'Hosted Ubuntu 1604' - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -#- template: linux-deps.yml - -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' -- script: ./.vsts/install_deps.sh - displayName: 'Install dependencies' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage - displayName: 'Set up virtual environment' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: ./venv/bin/python -m test.pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures - displayName: 'Tests with coverage' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) - displayName: 'Publish code coverage results' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml deleted file mode 100644 index d11a4f06e4e..00000000000 --- a/.vsts/linux-pr.yml +++ /dev/null @@ -1,84 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: 'Hosted Ubuntu 1604' - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -#- template: linux-deps.yml - -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: ./.vsts/install_deps.sh - displayName: 'Install dependencies' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -# Run patchcheck and fail if anything is discovered -- script: ./python Tools/scripts/patchcheck.py --travis true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: '$(system.pullRequest.targetBranch)-linux' - platform: linux - condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml deleted file mode 100644 index 69b619e4757..00000000000 --- a/.vsts/macos-pr.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: Hosted macOS - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - changes = $(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) - echo "Files changed:" - echo "$changes" - if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: '$(system.pullRequest.targetBranch)-macOS' - platform: macOS - condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml deleted file mode 100644 index 15aebeda407..00000000000 --- a/.vsts/windows-buildbot.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: Hosted VS2017 - parallel: 2 - matrix: - amd64: - buildOpt: -p x64 - outDirSuffix: amd64 - win32: - buildOpt: - outDirSuffix: win32 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Relocate build outputs outside of source directory to make cleaning faster - Py_IntDir: $(Build.BinariesDirectory)\obj - # UNDONE: Do not build to a different directory because of broken tests - Py_OutDir: $(Build.SourcesDirectory)\PCbuild - EXTERNAL_DIR: $(Build.BinariesDirectory)\externals - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" - displayName: 'Tests' - env: - PREFIX: $(Py_OutDir)\$(outDirSuffix) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' - mergeTestResults: true - testRunTitle: '$(Build.SourceBranchName)-$(outDirSuffix)' - platform: $(outDirSuffix) - condition: succeededOrFailed() diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml deleted file mode 100644 index 7134120d641..00000000000 --- a/.vsts/windows-pr.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Current docs for the syntax of this file are at: -# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md - -name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - -queue: - name: Hosted VS2017 - parallel: 2 - matrix: - amd64: - buildOpt: -p x64 - outDirSuffix: amd64 - win32: - buildOpt: - outDirSuffix: win32 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Relocate build outputs outside of source directory to make cleaning faster - Py_IntDir: $(Build.BinariesDirectory)\obj - # UNDONE: Do not build to a different directory because of broken tests - Py_OutDir: $(Build.SourcesDirectory)\PCbuild - EXTERNAL_DIR: $(Build.BinariesDirectory)\externals - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- powershell: | - git fetch -q origin $(System.PullRequest.TargetBranch) - if (-not (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD) | sls -NotMatch '(\.rst$)|(^Doc)|(^Misc)')) { - Write-Host 'Only docs were updated. Skipping build' - Write-Host '##vso[task.setvariable variable=DocOnly]true' - } - displayName: Detect doc-only changes - condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" - displayName: 'Tests' - env: - PREFIX: $(Py_OutDir)\$(outDirSuffix) - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' - mergeTestResults: true - testRunTitle: '$(System.PullRequest.TargetBranch)-$(outDirSuffix)' - platform: $(outDirSuffix) - condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 4871d60d756..bf5d64ceb29 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -436,18 +436,29 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Test flush() with the various options, using all the # different levels in order to provide more variations. sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', - 'Z_PARTIAL_FLUSH', 'Z_BLOCK'] + 'Z_PARTIAL_FLUSH'] + + ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) + # Z_BLOCK has a known failure prior to 1.2.5.3 + if ver >= (1, 2, 5, 3): + sync_opt.append('Z_BLOCK') + sync_opt = [getattr(zlib, opt) for opt in sync_opt if hasattr(zlib, opt)] data = HAMLET_SCENE * 8 for sync in sync_opt: for level in range(10): - obj = zlib.compressobj( level ) - a = obj.compress( data[:3000] ) - b = obj.flush( sync ) - c = obj.compress( data[3000:] ) - d = obj.flush() + try: + obj = zlib.compressobj( level ) + a = obj.compress( data[:3000] ) + b = obj.flush( sync ) + c = obj.compress( data[3000:] ) + d = obj.flush() + except: + print("Error for flush mode={}, level={}" + .format(sync, level)) + raise self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), data, ("Decompress failed: flush " "mode=%i, level=%i") % (sync, level))