From c9f480d2ccda9de46584cabe086f0acfa45f2faf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 6 Dec 2019 09:40:39 -0800 Subject: [PATCH] bpo-33125: Add support for building and releasing Windows ARM64 packages (GH-17480) Note that the support is not actually enabled yet, and so we won't be publishing these packages. However, for those who want to build it themselves (even by reusing the Azure Pipelines definition), it's now relatively easy to enable. --- .azure-pipelines/ci.yml | 2 +- .azure-pipelines/pr.yml | 5 ++- .../windows-release/build-steps.yml | 3 +- .../windows-release/layout-command.yml | 10 ++++- .../windows-release/msi-steps.yml | 1 + .../windows-release/stage-build.yml | 18 ++++++-- .../windows-release/stage-layout-embed.yml | 5 +++ .../windows-release/stage-layout-full.yml | 17 ++++++-- .../windows-release/stage-layout-msix.yml | 19 +++++--- .../windows-release/stage-layout-nuget.yml | 12 ++++-- .../windows-release/stage-pack-msix.yml | 10 +++++ .../windows-release/stage-pack-nuget.yml | 2 + .../stage-publish-nugetorg.yml | 5 +++ .../stage-publish-pythonorg.yml | 4 ++ .../windows-release/stage-sign.yml | 2 + .azure-pipelines/windows-steps.yml | 4 +- .../2019-11-14-08-57-50.bpo-33125.EN5MWS.rst | 1 + PC/layout/main.py | 24 +++++++++-- PC/layout/support/appxmanifest.py | 43 +++++++++++++++---- PC/layout/support/constants.py | 32 ++++++++++---- PC/layout/support/nuspec.py | 40 +++++++++-------- PC/layout/support/props.py | 22 +++++----- PCbuild/build.bat | 2 +- PCbuild/find_msbuild.bat | 2 +- PCbuild/pcbuild.proj | 2 +- PCbuild/prepare_libffi.bat | 7 ++- PCbuild/python_uwp.vcxproj | 32 ++++++++++++++ PCbuild/pythonw_uwp.vcxproj | 32 ++++++++++++++ 28 files changed, 285 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2019-11-14-08-57-50.bpo-33125.EN5MWS.rst diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index fe10e45b679..4c2f115cd9b 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -145,7 +145,7 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(Build.SourceBranchName)-win64' testRunPlatform: win64 - maxParallel: 2 + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 09209fc0c95..73d4f55b864 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -145,7 +145,10 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(System.PullRequest.TargetBranch)-win64' testRunPlatform: win64 - maxParallel: 2 + winarm64: + arch: arm64 + buildOpt: '-p arm64' + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml index d4563cd0d72..e2b6683f46e 100644 --- a/.azure-pipelines/windows-release/build-steps.yml +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -43,7 +43,7 @@ steps: - powershell: | $env:SigningCertificate = $null - .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default + python PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch) makecat "${env:CAT}.cdf" del "${env:CAT}.cdf" if (-not (Test-Path "${env:CAT}.cat")) { @@ -52,6 +52,7 @@ steps: displayName: 'Generate catalog' env: CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + PYTHON_HEXVERSION: $(VersionHex) - task: PublishPipelineArtifact@0 displayName: 'Publish binaries' diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml index 2dcd6ed26ca..406ccd859fa 100644 --- a/.azure-pipelines/windows-release/layout-command.yml +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -1,12 +1,20 @@ steps: +- task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(HostArch)' + condition: and(succeeded(), variables['HostArch']) + inputs: + artifactName: bin_$(HostArch) + targetPath: $(Build.BinariesDirectory)\bin_$(HostArch) + - powershell: > Write-Host ( '##vso[task.setvariable variable=LayoutCmd]& - "{0}\bin\python.exe" + "$(Python)" "{1}\PC\layout" -vv --source "{1}" --build "{0}\bin" + --arch "$(Name)" --temp "{0}\layout-temp" --include-cat "{0}\bin\python.cat" --doc-build "{0}\doc"' diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml index f7bff162f8e..a460eb1bac8 100644 --- a/.azure-pipelines/windows-release/msi-steps.yml +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -54,6 +54,7 @@ steps: - powershell: | copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) - script: | call Tools\msi\get_externals.bat diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml index c98576ef970..60d72b282d1 100644 --- a/.azure-pipelines/windows-release/stage-build.yml +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -16,14 +16,16 @@ jobs: env: BUILDDIR: $(Build.BinariesDirectory)\Doc - #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm" - # displayName: 'Cheat at building CHM docs' - - script: Doc\make.bat htmlhelp displayName: 'Build CHM docs' env: BUILDDIR: $(Build.BinariesDirectory)\Doc + #- powershell: | + # mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp" + # iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + - task: CopyFiles@2 displayName: 'Assemble artifact: Doc' inputs: @@ -65,6 +67,16 @@ jobs: Arch: amd64 Platform: x64 Configuration: Debug + arm64: + Name: arm64 + Arch: arm64 + Platform: ARM64 + Configuration: Release + arm64_d: + Name: arm64_d + Arch: arm64 + Platform: ARM64 + Configuration: Debug steps: - template: ./build-steps.yml diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml index 09857ff676b..3306e1cbc49 100644 --- a/.azure-pipelines/windows-release/stage-layout-embed.yml +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -19,6 +19,11 @@ jobs: Name: amd64 Python: $(Build.BinariesDirectory)\bin\python.exe PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) steps: - template: ./checkout.yml diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml index 12c34723901..78bc1b3975e 100644 --- a/.azure-pipelines/windows-release/stage-layout-full.yml +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -13,11 +13,18 @@ jobs: matrix: win32: Name: win32 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 amd64: Name: amd64 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe PYTHONHOME: $(Build.SourcesDirectory) steps: @@ -43,13 +50,15 @@ jobs: - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) inputs: artifactName: tcltk_lib_$(Name) targetPath: $(Build.BinariesDirectory)\tcltk_lib - powershell: | - copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) - template: ./layout-command.yml @@ -57,7 +66,7 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default displayName: 'Generate full layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + TCL_LIBRARY: $(TclLibrary) - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_full_$(Name)' diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml index ba86392f3ec..60a5c9ea543 100644 --- a/.azure-pipelines/windows-release/stage-layout-msix.yml +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -12,11 +12,18 @@ jobs: matrix: #win32: # Name: win32 - # Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + # Python: $(Build.BinariesDirectory)\bin\python.exe # PYTHONHOME: $(Build.SourcesDirectory) + # TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 amd64: Name: amd64 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe PYTHONHOME: $(Build.SourcesDirectory) steps: @@ -36,13 +43,15 @@ jobs: - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) inputs: artifactName: tcltk_lib_$(Name) targetPath: $(Build.BinariesDirectory)\tcltk_lib - powershell: | - copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) - template: ./layout-command.yml @@ -51,7 +60,7 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile displayName: 'Generate store APPX layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + TCL_LIBRARY: $(TclLibrary) - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_appxstore_$(Name)' @@ -79,7 +88,7 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests displayName: 'Generate sideloading APPX layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + TCL_LIBRARY: $(TclLibrary) - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_appx_$(Name)' diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml index 7954c4547f5..7e20f895303 100644 --- a/.azure-pipelines/windows-release/stage-layout-nuget.yml +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -13,11 +13,16 @@ jobs: matrix: win32: Name: win32 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe PYTHONHOME: $(Build.SourcesDirectory) amd64: Name: amd64 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe PYTHONHOME: $(Build.SourcesDirectory) steps: @@ -32,14 +37,13 @@ jobs: - powershell: | copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) - template: ./layout-command.yml - powershell: | $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget displayName: 'Generate nuget layout' - env: - TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_nuget_$(Name)' diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml index eebc63fb880..f17ba9628e2 100644 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -20,6 +20,16 @@ jobs: Artifact: appxstore Suffix: -store Upload: true + arm64: + Name: arm64 + Artifact: appx + Suffix: + ShouldSign: true + arm64_store: + Name: arm64 + Artifact: appxstore + Suffix: -store + Upload: true steps: - template: ./checkout.yml diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml index f59bbe9b39a..34619fc5fdc 100644 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -15,6 +15,8 @@ jobs: Name: amd64 win32: Name: win32 + arm64: + Name: arm64 steps: - checkout: none diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml index 570cdb3ec57..b78bd493a0f 100644 --- a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -31,6 +31,11 @@ jobs: buildVersionToDownload: specific buildId: $(BuildToPublish) + - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\nuget' + condition: and(succeeded(), not(variables['PublishArmPackages'])) + - task: NuGetCommand@2 displayName: Push packages condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml index 2dd354a8c27..8c95f1b950c 100644 --- a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -39,6 +39,10 @@ jobs: artifactName: embed downloadPath: $(Build.BinariesDirectory) + - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\embed' + condition: and(succeeded(), not(variables['PublishArmPackages'])) - task: DownloadPipelineArtifact@1 displayName: 'Download artifact from $(BuildToPublish): Doc' diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml index 2307c6c9c8f..a0adc058122 100644 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -19,6 +19,8 @@ jobs: Name: win32 amd64: Name: amd64 + arm64: + Name: arm64 steps: - template: ./checkout.yml diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml index 794a23a5d77..f502c40637c 100644 --- a/.azure-pipelines/windows-steps.yml +++ b/.azure-pipelines/windows-steps.yml @@ -19,9 +19,11 @@ steps: - script: python.bat -m test.pythoninfo displayName: 'Display build info' + condition: and(succeeded(), variables['testRunPlatform']) - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test" displayName: 'Tests' + condition: and(succeeded(), variables['testRunPlatform']) env: PREFIX: $(Py_OutDir)\$(arch) @@ -32,4 +34,4 @@ steps: mergeTestResults: true testRunTitle: $(testRunTitle) platform: $(testRunPlatform) - condition: succeededOrFailed() + condition: and(succeededOrFailed(), variables['testRunPlatform']) diff --git a/Misc/NEWS.d/next/Windows/2019-11-14-08-57-50.bpo-33125.EN5MWS.rst b/Misc/NEWS.d/next/Windows/2019-11-14-08-57-50.bpo-33125.EN5MWS.rst new file mode 100644 index 00000000000..0bc98c1a2fa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2019-11-14-08-57-50.bpo-33125.EN5MWS.rst @@ -0,0 +1 @@ +Add support for building and releasing Windows ARM64 packages. diff --git a/PC/layout/main.py b/PC/layout/main.py index 3ca49d08e8f..305cb517d0f 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -285,14 +285,13 @@ def _compile_one_py(src, dest, name, optimize, checked=True): log_warning("Failed to compile {}", src) return None + # name argument added to address bpo-37641 def _py_temp_compile(src, name, ns, dest_dir=None, checked=True): if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: return None dest = (dest_dir or ns.temp) / (src.stem + ".pyc") - return _compile_one_py( - src, dest, name, optimize=2, checked=checked - ) + return _compile_one_py(src, dest, name, optimize=2, checked=checked) def _write_to_zip(zf, dest, src, ns, checked=True): @@ -496,6 +495,13 @@ def main(): parser.add_argument( "-b", "--build", metavar="dir", help="Specify the build directory", type=Path ) + parser.add_argument( + "--arch", + metavar="architecture", + help="Specify the target architecture", + type=str, + default=None, + ) parser.add_argument( "--doc-build", metavar="dir", @@ -587,6 +593,8 @@ def main(): ns.doc_build = (Path.cwd() / ns.doc_build).resolve() if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() + if not ns.arch: + ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" if ns.copy and not ns.copy.is_absolute(): ns.copy = (Path.cwd() / ns.copy).resolve() @@ -602,6 +610,7 @@ def main(): Source: {ns.source} Build: {ns.build} Temp: {ns.temp} +Arch: {ns.arch} Copy to: {ns.copy} Zip to: {ns.zip} @@ -609,6 +618,15 @@ Catalog: {ns.catalog}""", ns=ns, ) + if ns.arch not in ("win32", "amd64", "arm32", "arm64"): + log_error("--arch is not a valid value (win32, amd64, arm32, arm64)") + return 4 + if ns.arch in ("arm32", "arm64"): + for n in ("include_idle", "include_tcltk"): + if getattr(ns, n): + log_warning(f"Disabling --{n.replace('_', '-')} on unsupported platform") + setattr(ns, n, False) + if ns.include_idle and not ns.include_tcltk: log_warning("Assuming --include-tcltk to support --include-idle") ns.include_tcltk = True diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index de5813a2536..9e008f793cf 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -28,7 +28,14 @@ APPX_DATA = dict( ), DisplayName="Python {}".format(VER_DOT), Description="The Python {} runtime and console.".format(VER_DOT), - ProcessorArchitecture="x64" if IS_X64 else "x86", +) + +APPX_PLATFORM_DATA = dict( + _keys=("ProcessorArchitecture",), + win32=("x86",), + amd64=("x64",), + arm32=("arm",), + arm64=("arm64",), ) PYTHON_VE_DATA = dict( @@ -65,7 +72,7 @@ IDLE_VE_DATA = dict( BackgroundColor="transparent", ) -PY_PNG = '_resources/py.png' +PY_PNG = "_resources/py.png" APPXMANIFEST_NS = { "": "http://schemas.microsoft.com/appx/manifest/foundation/windows10", @@ -147,18 +154,22 @@ RESOURCES_XML_TEMPLATE = r""" 1: yield str(fullkey), None, None yield from _get_registry_entries(ns, fullkey, value) + elif value is SPECIAL_LOOKUP: + if key == "SysArchitecture": + return { + "win32": "32bit", + "amd64": "64bit", + "arm32": "32bit", + "arm64": "64bit", + }[ns.arch] + else: + raise ValueError(f"Key '{key}' unhandled for special lookup") elif len(r.parts) > 1: yield str(r), key, value @@ -376,14 +397,18 @@ def get_appxmanifest(ns): NS = APPXMANIFEST_NS QN = ET.QName + data = dict(APPX_DATA) + for k, v in zip(APPX_PLATFORM_DATA["_keys"], APPX_PLATFORM_DATA[ns.arch]): + data[k] = v + node = xml.find("m:Identity", NS) for k in node.keys(): - value = APPX_DATA.get(k) + value = data.get(k) if value: node.set(k, value) for node in xml.find("m:Properties", NS): - value = APPX_DATA.get(node.tag.rpartition("}")[2]) + value = data.get(node.tag.rpartition("}")[2]) if value: node.text = value @@ -405,7 +430,7 @@ def get_appxmanifest(ns): ["python", "python{}".format(VER_MAJOR), "python{}".format(VER_DOT)], PYTHON_VE_DATA, "console", - ("python.file", [".py"], '"%1"', 'Python File', PY_PNG), + ("python.file", [".py"], '"%1"', "Python File", PY_PNG), ) add_application( @@ -416,7 +441,7 @@ def get_appxmanifest(ns): ["pythonw", "pythonw{}".format(VER_MAJOR), "pythonw{}".format(VER_DOT)], PYTHONW_VE_DATA, "windows", - ("python.windowedfile", [".pyw"], '"%1"', 'Python File (no console)', PY_PNG), + ("python.windowedfile", [".pyw"], '"%1"', "Python File (no console)", PY_PNG), ) if ns.include_pip and ns.include_launchers: @@ -428,7 +453,7 @@ def get_appxmanifest(ns): ["pip", "pip{}".format(VER_MAJOR), "pip{}".format(VER_DOT)], PIP_VE_DATA, "console", - ("python.wheel", [".whl"], 'install "%1"', 'Python Wheel'), + ("python.wheel", [".whl"], 'install "%1"', "Python Wheel"), ) if ns.include_idle and ns.include_launchers: diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index d76fa3bbf3b..a8647631e9b 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -5,15 +5,31 @@ Constants for generating the layout. __author__ = "Steve Dower " __version__ = "3.8" +import os +import re import struct import sys -VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = struct.pack(">i", sys.hexversion) + +def _unpack_hexversion(): + try: + hexversion = int(os.getenv("PYTHON_HEXVERSION"), 16) + except (TypeError, ValueError): + hexversion = sys.hexversion + return struct.pack(">i", sys.hexversion) + + +def _get_suffix(field4): + name = {0xA0: "a", 0xB0: "b", 0xC0: "c"}.get(field4 & 0xF0, "") + if name: + serial = field4 & 0x0F + return f"{name}{serial}" + return "" + + +VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() +VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 -VER_NAME = {"alpha": "a", "beta": "b", "candidate": "rc"}.get( - sys.version_info.releaselevel, "" -) -VER_SERIAL = sys.version_info.serial if VER_NAME else "" VER_DOT = "{}.{}".format(VER_MAJOR, VER_MINOR) PYTHON_DLL_NAME = "python{}{}.dll".format(VER_MAJOR, VER_MINOR) @@ -21,8 +37,6 @@ PYTHON_STABLE_DLL_NAME = "python{}.dll".format(VER_MAJOR) PYTHON_ZIP_NAME = "python{}{}.zip".format(VER_MAJOR, VER_MINOR) PYTHON_PTH_NAME = "python{}{}._pth".format(VER_MAJOR, VER_MINOR) -PYTHON_CHM_NAME = "python{}{}{}{}{}.chm".format( - VER_MAJOR, VER_MINOR, VER_MICRO, VER_NAME, VER_SERIAL +PYTHON_CHM_NAME = "python{}{}{}{}.chm".format( + VER_MAJOR, VER_MINOR, VER_MICRO, VER_SUFFIX ) - -IS_X64 = sys.maxsize > 2 ** 32 diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py index ba26ff337e9..b85095c555f 100644 --- a/PC/layout/support/nuspec.py +++ b/PC/layout/support/nuspec.py @@ -13,25 +13,21 @@ PYTHON_NUSPEC_NAME = "python.nuspec" NUSPEC_DATA = { "PYTHON_TAG": VER_DOT, "PYTHON_VERSION": os.getenv("PYTHON_NUSPEC_VERSION"), - "PYTHON_BITNESS": "64-bit" if IS_X64 else "32-bit", - "PACKAGENAME": os.getenv("PYTHON_NUSPEC_PACKAGENAME"), - "PACKAGETITLE": os.getenv("PYTHON_NUSPEC_PACKAGETITLE"), "FILELIST": r' ', } +NUSPEC_PLATFORM_DATA = dict( + _keys=("PYTHON_BITNESS", "PACKAGENAME", "PACKAGETITLE"), + win32=("32-bit", "pythonx86", "Python (32-bit)"), + amd64=("64-bit", "python", "Python"), + arm32=("ARM", "pythonarm", "Python (ARM)"), + arm64=("ARM64", "pythonarm64", "Python (ARM64)"), +) + if not NUSPEC_DATA["PYTHON_VERSION"]: - if VER_NAME: - NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}-{}{}".format( - VER_DOT, VER_MICRO, VER_NAME, VER_SERIAL - ) - else: - NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}".format(VER_DOT, VER_MICRO) - -if not NUSPEC_DATA["PACKAGETITLE"]: - NUSPEC_DATA["PACKAGETITLE"] = "Python" if IS_X64 else "Python (32-bit)" - -if not NUSPEC_DATA["PACKAGENAME"]: - NUSPEC_DATA["PACKAGENAME"] = "python" if IS_X64 else "pythonx86" + NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}{}{}".format( + VER_DOT, VER_MICRO, "-" if VER_SUFFIX else "", VER_SUFFIX + ) FILELIST_WITH_PROPS = r""" """ @@ -56,11 +52,21 @@ NUSPEC_TEMPLATE = r""" """ +def _get_nuspec_data_overrides(ns): + for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[ns.arch]): + ev = os.getenv("PYTHON_NUSPEC_" + k) + if ev: + yield k, ev + yield k, v + + def get_nuspec_layout(ns): if ns.include_all or ns.include_nuspec: - data = NUSPEC_DATA + data = dict(NUSPEC_DATA) + for k, v in _get_nuspec_data_overrides(ns): + if not data.get(k): + data[k] = v if ns.include_all or ns.include_props: - data = dict(data) data["FILELIST"] = FILELIST_WITH_PROPS nuspec = NUSPEC_TEMPLATE.format_map(data) yield "python.nuspec", ("python.nuspec", nuspec.encode("utf-8")) diff --git a/PC/layout/support/props.py b/PC/layout/support/props.py index 4d3b06195f6..b1560b52447 100644 --- a/PC/layout/support/props.py +++ b/PC/layout/support/props.py @@ -18,15 +18,9 @@ PROPS_DATA = { } if not PROPS_DATA["PYTHON_VERSION"]: - if VER_NAME: - PROPS_DATA["PYTHON_VERSION"] = "{}.{}-{}{}".format( - VER_DOT, VER_MICRO, VER_NAME, VER_SERIAL - ) - else: - PROPS_DATA["PYTHON_VERSION"] = "{}.{}".format(VER_DOT, VER_MICRO) - -if not PROPS_DATA["PYTHON_PLATFORM"]: - PROPS_DATA["PYTHON_PLATFORM"] = "x64" if IS_X64 else "Win32" + PROPS_DATA["PYTHON_VERSION"] = "{}.{}{}{}".format( + VER_DOT, VER_MICRO, "-" if VER_SUFFIX else "", VER_SUFFIX + ) PROPS_DATA["PYTHON_TARGET"] = "_GetPythonRuntimeFilesDependsOn{}{}_{}".format( VER_MAJOR, VER_MINOR, PROPS_DATA["PYTHON_PLATFORM"] @@ -94,5 +88,13 @@ PROPS_TEMPLATE = r""" def get_props_layout(ns): if ns.include_all or ns.include_props: # TODO: Filter contents of props file according to included/excluded items - props = PROPS_TEMPLATE.format_map(PROPS_DATA) + d = dict(PROPS_DATA) + if not d.get("PYTHON_PLATFORM"): + d["PYTHON_PLATFORM"] = { + "win32": "Win32", + "amd64": "X64", + "arm32": "ARM", + "arm64": "ARM64", + }[ns.arch] + props = PROPS_TEMPLATE.format_map(d) yield "python.props", ("python.props", props.encode("utf-8")) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index bce599329e7..623409c24ec 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -57,7 +57,7 @@ set conf=Release set target=Build set dir=%~dp0 set parallel=/m -set verbose=/nologo /v:m +set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat index a2810f09c45..bc9d00c22c8 100644 --- a/PCbuild/find_msbuild.bat +++ b/PCbuild/find_msbuild.bat @@ -32,7 +32,7 @@ @rem VS 2017 and later provide vswhere.exe, which can be used @if not exist "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" goto :skip_vswhere @set _Py_MSBuild_Root= -@for /F "tokens=*" %%i in ('"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -property installationPath -latest') DO @(set _Py_MSBuild_Root=%%i\MSBuild) +@for /F "tokens=*" %%i in ('"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -property installationPath -latest -prerelease') DO @(set _Py_MSBuild_Root=%%i\MSBuild) @if not defined _Py_MSBuild_Root goto :skip_vswhere @for %%j in (Current 15.0) DO @if exist "%_Py_MSBuild_Root%\%%j\Bin\msbuild.exe" (set MSBUILD="%_Py_MSBuild_Root%\%%j\Bin\msbuild.exe") @set _Py_MSBuild_Root= diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 35f173ff864..22a9eed18d4 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -60,7 +60,7 @@ - + diff --git a/PCbuild/prepare_libffi.bat b/PCbuild/prepare_libffi.bat index c65a5f7010b..f41ba83379a 100644 --- a/PCbuild/prepare_libffi.bat +++ b/PCbuild/prepare_libffi.bat @@ -93,7 +93,10 @@ echo LIBFFI_SOURCE: %LIBFFI_SOURCE% echo MSVCC : %MSVCC% echo. -if not exist Makefile.in (%SH% -lc "(cd $LIBFFI_SOURCE; ./autogen.sh;)") +if not exist Makefile.in ( + %SH% -lc "(cd $LIBFFI_SOURCE; ./autogen.sh;)" + if errorlevel 1 exit /B 1 +) if "%BUILD_X64%"=="1" call :BuildOne x64 x86_64-w64-cygwin x86_64-w64-cygwin if "%BUILD_X86%"=="1" call :BuildOne x86 i686-pc-cygwin i686-pc-cygwin @@ -158,11 +161,13 @@ echo ================================================================ echo Configure the build to generate fficonfig.h and ffi.h echo ================================================================ %SH% -lc "(cd $OLDPWD; ./configure CC='%MSVCC% %ASSEMBLER% %BUILD_PDB%' CXX='%MSVCC% %ASSEMBLER% %BUILD_PDB%' LD='link' CPP='cl -nologo -EP' CXXCPP='cl -nologo -EP' CPPFLAGS='-DFFI_BUILDING_DLL' %BUILD_NOOPT% NM='dumpbin -symbols' STRIP=':' --build=$BUILD --host=$HOST;)" +if errorlevel 1 exit /B %ERRORLEVEL% echo ================================================================ echo Building libffi echo ================================================================ %SH% -lc "(cd $OLDPWD; export PATH=/usr/bin:$PATH; cp src/%SRC_ARCHITECTURE%/ffitarget.h include; make; find .;)" +if errorlevel 1 exit /B %ERRORLEVEL% REM Tests are not needed to produce artifacts if "%LIBFFI_TEST%" EQU "1" ( diff --git a/PCbuild/python_uwp.vcxproj b/PCbuild/python_uwp.vcxproj index 14e138cbed3..5ff120a0da3 100644 --- a/PCbuild/python_uwp.vcxproj +++ b/PCbuild/python_uwp.vcxproj @@ -1,6 +1,14 @@  + + Debug + ARM + + + Debug + ARM64 + Debug Win32 @@ -9,6 +17,14 @@ Debug x64 + + PGInstrument + ARM + + + PGInstrument + ARM64 + PGInstrument Win32 @@ -17,6 +33,14 @@ PGInstrument x64 + + PGUpdate + ARM + + + PGUpdate + ARM64 + PGUpdate Win32 @@ -25,6 +49,14 @@ PGUpdate x64 + + Release + ARM + + + Release + ARM64 + Release Win32 diff --git a/PCbuild/pythonw_uwp.vcxproj b/PCbuild/pythonw_uwp.vcxproj index e2c01710498..828d0d1ccac 100644 --- a/PCbuild/pythonw_uwp.vcxproj +++ b/PCbuild/pythonw_uwp.vcxproj @@ -1,6 +1,14 @@  + + Debug + ARM + + + Debug + ARM64 + Debug Win32 @@ -9,6 +17,14 @@ Debug x64 + + PGInstrument + ARM + + + PGInstrument + ARM64 + PGInstrument Win32 @@ -17,6 +33,14 @@ PGInstrument x64 + + PGUpdate + ARM + + + PGUpdate + ARM64 + PGUpdate Win32 @@ -25,6 +49,14 @@ PGUpdate x64 + + Release + ARM + + + Release + ARM64 + Release Win32