bpo-30450: Pull Windows dependencies from GitHub rather than svn (GH-1783)

The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`.  This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio.

Also fixes an outdated note about _lzma in PCbuild/readme.txt

PCbuild/get_external.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import zipfile
+from urllib.request import urlretrieve
+
+
+def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
+    repo = f'cpython-{"bin" if binary else "source"}-deps'
+    url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
+    reporthook = None
+    if verbose:
+        reporthook = print
+    zip_dir.mkdir(parents=True, exist_ok=True)
+    filename, headers = urlretrieve(
+        url,
+        zip_dir / f'{commit_hash}.zip',
+        reporthook=reporthook,
+    )
+    return filename
+
+
+def extract_zip(externals_dir, zip_path):
+    with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+        zf.extractall(os.fspath(externals_dir))
+        return externals_dir / zf.namelist()[0].split('/')[0]
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('-v', '--verbose', action='store_true')
+    p.add_argument('-b', '--binary', action='store_true',
+                   help='Is the dependency in the binary repo?')
+    p.add_argument('-O', '--organization',
+                   help='Organization owning the deps repos', default='python')
+    p.add_argument('-e', '--externals-dir', type=pathlib.Path,
+                   help='Directory in which to store dependencies',
+                   default=pathlib.Path(__file__).parent.parent / 'externals')
+    p.add_argument('tag',
+                   help='tag of the dependency')
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    zip_path = fetch_zip(
+        args.tag,
+        args.externals_dir / 'zips',
+        org=args.organization,
+        binary=args.binary,
+        verbose=args.verbose,
+    )
+    final_name = args.externals_dir / args.tag
+    extract_zip(args.externals_dir, zip_path).replace(final_name)
+
+
+if __name__ == '__main__':
+    main()

PCbuild/get_externals.bat

@@ -2,58 +2,59 @@
 setlocal
 rem Simple script to fetch source for external libraries
 
-if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
+if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
-pushd "%~dp0..\externals"
+if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
+if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
+if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
 
-if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+set DO_FETCH=true
+set DO_CLEAN=false
 
-rem Optionally clean up first.  Be warned that this can be very destructive!
+:CheckOpts
-if not "%1"=="" (
+if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
-    for %%c in (-c --clean --clean-only) do (
+if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
-        if "%1"=="%%c" goto clean
+if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
-    )
+if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
-    goto usage
+if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
-)
+if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
-goto fetch
+if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
+if "x%~1" NEQ "x" goto usage
 
+if "%DO_CLEAN%"=="false" goto fetch
 :clean
 echo.Cleaning up external libraries.
-for /D %%d in (
+if exist "%EXTERNALS_DIR%" (
-               bzip2-*
+    rem Sometimes this fails the first time; try it twice
-               db-*
+    rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
-               nasm-*
-               openssl-*
-               tcl-*
-               tcltk*
-               tk-*
-               tix-*
-               sqlite-*
-               xz-*
-               ) do (
-    echo.Removing %%d
-    rmdir /s /q %%d
-)
-if "%1"=="--clean-only" (
-    goto end
 )
 
+if "%DO_FETCH%"=="false" goto end
 :fetch
-rem Fetch current versions
 
-svn --version > nul 2>&1
+if "%ORG%"=="" (set ORG=python)
-if ERRORLEVEL 9009 (
+
-    echo.svn.exe must be on your PATH.
+if "%PYTHON_FOR_BUILD%"=="" (
-    echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
+    echo Checking for installed python...
-    echo.command line tools option.
+    py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
-    popd
+)
-    exit /b 1
+if "%PYTHON_FOR_BUILD%"=="" (
+    if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
+    if NOT exist "%NUGET%" (
+        echo Downloading nuget...
+        rem NB: Must use single quotes around NUGET here, NOT double!
+        rem Otherwise, a space in the path would break things
+        powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
+    )
+    echo Installing Python via nuget...
+    "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
+    rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+    set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
 )
 
 echo.Fetching external libraries...
 
 set libraries=
 set libraries=%libraries%                                    bzip2-1.0.6
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     nasm-2.11.06
 if NOT "%IncludeSSL%"=="false" set libraries=%libraries%     openssl-1.0.2k
 set libraries=%libraries%                                    sqlite-3.14.2.0
 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0
@@ -62,43 +63,48 @@ if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6
 set libraries=%libraries%                                    xz-5.2.2
 
 for %%e in (%libraries%) do (
-    if exist %%e (
+    if exist "%EXTERNALS_DIR%\%%e" (
         echo.%%e already exists, skipping.
     ) else (
         echo.Fetching %%e...
-        svn export -q %SVNROOT%%%e
+        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
     )
 )
 
+echo.Fetching external binaries...
+
+set binaries=
+set binaries=%binaries%
+if NOT "%IncludeSSL%"=="false" set binaries=%binaries%     nasm-2.11.06
+
+for %%b in (%binaries%) do (
+    if exist "%EXTERNALS_DIR%\%%b" (
+        echo.%%b already exists, skipping.
+    ) else (
+        echo.Fetching %%b...
+        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
+    )
+)
+
+echo Finished.
 goto end
 
 :usage
-echo.invalid argument: %1
+echo.Valid options: -c, --clean, --clean-only, --organization, --python,
-echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.--no-tkinter, --no-openssl
 echo.
-echo.Pull all sources necessary for compiling optional extension modules
+echo.Pull all sources and binaries necessary for compiling optional extension
-echo.that rely on external libraries.  Requires svn.exe to be on your PATH
+echo.modules that rely on external libraries.
-echo.and pulls sources from %SVNROOT%.
 echo.
-echo.Use the -c or --clean option to clean up all external library sources
+echo.The --organization option determines which github organization to download
-echo.before pulling in the current versions.
+echo.from, the --python option determines which Python 3.6+ interpreter to use
+echo.with PCbuild\get_external.py.
+echo.
+echo.Use the -c or --clean option to remove the entire externals directory.
 echo.
 echo.Use the --clean-only option to do the same cleaning, without pulling in
 echo.anything new.
 echo.
-echo.Only the first argument is checked, all others are ignored.
-echo.
-echo.**WARNING**: the cleaning options unconditionally remove any directory
-echo.that is a child of
-echo.   %CD%
-echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
-echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
-echo.to be very destructive if you are not aware of what it is doing.  Use with
-echo.caution!
-popd
 exit /b -1
 
-
 :end
-echo Finished.
-popd

PCbuild/readme.txt

@@ -2,9 +2,11 @@ Quick Start Guide
 -----------------
 
 1.  Install Microsoft Visual Studio 2015, any edition.
-2.  Install Subversion, and make sure 'svn.exe' is on your PATH.
+1a. Optionally install Python 3.6 or later.  If not installed,
-3.  Run "build.bat -e" to build Python in 32-bit Release configuration.
+    get_externals.bat (build.bat -e) will download and use Python via
-4.  (Optional, but recommended) Run the test suite with "rt.bat -q".
+    NuGet.
+2.  Run "build.bat -e" to build Python in 32-bit Release configuration.
+3.  (Optional, but recommended) Run the test suite with "rt.bat -q".
 
 
 Building Python using Microsoft Visual C++
@@ -164,8 +166,7 @@ _bz2
     Homepage:
         http://www.bzip.org/
 _lzma
-    Python wrapper for the liblzma compression library, using pre-built
+    Python wrapper for version 5.2.2 of the liblzma compression library
-    binaries of XZ Utils version 5.0.5
     Homepage:
         http://tukaani.org/xz/
 _ssl
@@ -236,9 +237,16 @@ order to download the relevant source files for each project before they
 can be built.  However, a simple script is provided to make this as
 painless as possible, called "get_externals.bat" and located in this
 directory.  This script extracts all the external sub-projects from
-    http://svn.python.org/projects/external
+    https://github.com/python/cpython-source-deps
-via Subversion (so you'll need svn.exe on your PATH) and places them
+and
-in ..\externals (relative to this directory).
+    https://github.com/python/cpython-bin-deps
+via a Python script called "get_external.py", located in this directory.
+If Python 3.6 or later is not available via the "py.exe" launcher, the
+path or command to use for Python can be provided in the PYTHON_FOR_BUILD
+environment variable, or get_externals.bat will download the latest
+version of NuGet and use it to download the latest "pythonx86" package
+for use with get_external.py.  Everything downloaded by these scripts is
+stored in ..\externals (relative to this directory).
 
 It is also possible to download sources from each project's homepage,
 though you may have to change folder names or pass the names to MSBuild

Tools/msi/get_externals.bat

@@ -1,27 +1,107 @@
 @echo off
 setlocal
-rem Simple script to fetch source for external tools
+rem Simple script to fetch source for external libraries
 
-where /Q svn
+set HERE=%~dp0
-if ERRORLEVEL 1 (
+if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\)
-    echo.svn.exe must be on your PATH to get external tools.
+if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer)
-    echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
+if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe)
-    echo.command line tools option.
+if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
-    popd
+
-    exit /b 1
+set DO_FETCH=true
+set DO_CLEAN=false
+
+:CheckOpts
+if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
+if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
+if "x%~1" NEQ "x" goto usage
+
+if "%DO_CLEAN%"=="false" goto fetch
+:clean
+echo.Cleaning up external libraries.
+if exist "%EXTERNALS_DIR%" (
+    rem Sometimes this fails the first time; try it twice
+    rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
 )
 
-if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals"
+if "%DO_FETCH%"=="false" goto end
-pushd "%~dp0..\..\externals"
+:fetch
 
-if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+if "%ORG%"=="" (set ORG=python)
 
-if not exist "windows-installer\.svn" (
+if "%PYTHON_FOR_BUILD%"=="" (
-    echo.Checking out installer dependencies to %CD%\windows-installer
+    echo Checking for installed python...
-    svn co %SVNROOT%windows-installer
+    py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
-) else (
+)
-    echo.Updating installer dependencies in %CD%\windows-installer
+if "%PYTHON_FOR_BUILD%"=="" (
-    svn up windows-installer
+    if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
+    if NOT exist "%NUGET%" (
+        echo Downloading nuget...
+        rem NB: Must use single quotes around NUGET here, NOT double!
+        rem Otherwise, a space in the path would break things
+        powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
+    )
+    echo Installing Python via nuget...
+    "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
+    rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+    set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
 )
 
-popd
+echo.Fetching external libraries...
+
+set libraries=
+
+for %%e in (%libraries%) do (
+    if exist "%EXTERNALS_DIR%\%%e" (
+        echo.%%e already exists, skipping.
+    ) else (
+        echo.Fetching %%e...
+        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
+    )
+)
+
+echo.Fetching external tools...
+
+set binaries=
+rem We always use whatever's latest in the repo for these
+set binaries=%binaries%     binutils
+set binaries=%binaries%     gpg
+set binaries=%binaries%     htmlhelp
+set binaries=%binaries%     nuget
+set binaries=%binaries%     redist
+set binaries=%binaries%     wix
+
+for %%b in (%binaries%) do (
+    if exist "%EXTERNALS_DIR%\%%b" (
+        echo.%%b already exists, skipping.
+    ) else (
+        echo.Fetching %%b...
+        %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
+    )
+)
+
+echo Finished.
+goto end
+
+:usage
+echo.Valid options: -c, --clean, --clean-only, --organization, --python,
+echo.--no-tkinter, --no-openssl
+echo.
+echo.Pull all sources and binaries necessary for compiling optional extension
+echo.modules that rely on external libraries.
+echo.
+echo.The --organization option determines which github organization to download
+echo.from, the --python option determines which Python 3.6+ interpreter to use
+echo.with PCbuild\get_external.py.
+echo.
+echo.Use the -c or --clean option to remove the entire externals directory.
+echo.
+echo.Use the --clean-only option to do the same cleaning, without pulling in
+echo.anything new.
+echo.
+exit /b -1
+
+:end