bpo-30450: Pull Windows dependencies from GitHub rather than svn (GH-1783)
The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`. This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio. Also fixes an outdated note about _lzma in PCbuild/readme.txt
This commit is contained in:
Zachary Ware
GitHub
parent
7a801839e9
commit
51599e2bdd
|
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import zipfile
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
|
||||
def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
|
||||
repo = f'cpython-{"bin" if binary else "source"}-deps'
|
||||
url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
|
||||
reporthook = None
|
||||
if verbose:
|
||||
reporthook = print
|
||||
zip_dir.mkdir(parents=True, exist_ok=True)
|
||||
filename, headers = urlretrieve(
|
||||
url,
|
||||
zip_dir / f'{commit_hash}.zip',
|
||||
reporthook=reporthook,
|
||||
)
|
||||
return filename
|
||||
|
||||
|
||||
def extract_zip(externals_dir, zip_path):
|
||||
with zipfile.ZipFile(os.fspath(zip_path)) as zf:
|
||||
zf.extractall(os.fspath(externals_dir))
|
||||
return externals_dir / zf.namelist()[0].split('/')[0]
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('-v', '--verbose', action='store_true')
|
||||
p.add_argument('-b', '--binary', action='store_true',
|
||||
help='Is the dependency in the binary repo?')
|
||||
p.add_argument('-O', '--organization',
|
||||
help='Organization owning the deps repos', default='python')
|
||||
p.add_argument('-e', '--externals-dir', type=pathlib.Path,
|
||||
help='Directory in which to store dependencies',
|
||||
default=pathlib.Path(__file__).parent.parent / 'externals')
|
||||
p.add_argument('tag',
|
||||
help='tag of the dependency')
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
zip_path = fetch_zip(
|
||||
args.tag,
|
||||
args.externals_dir / 'zips',
|
||||
org=args.organization,
|
||||
binary=args.binary,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
final_name = args.externals_dir / args.tag
|
||||
extract_zip(args.externals_dir, zip_path).replace(final_name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -2,58 +2,59 @@
|
|||
setlocal
|
||||
rem Simple script to fetch source for external libraries
|
||||
|
||||
if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
|
||||
pushd "%~dp0..\externals"
|
||||
if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
|
||||
if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
|
||||
if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
|
||||
if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
|
||||
|
||||
if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
|
||||
set DO_FETCH=true
|
||||
set DO_CLEAN=false
|
||||
|
||||
rem Optionally clean up first. Be warned that this can be very destructive!
|
||||
if not "%1"=="" (
|
||||
for %%c in (-c --clean --clean-only) do (
|
||||
if "%1"=="%%c" goto clean
|
||||
)
|
||||
goto usage
|
||||
)
|
||||
goto fetch
|
||||
:CheckOpts
|
||||
if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
|
||||
if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
|
||||
if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
|
||||
if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
|
||||
if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
|
||||
if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
|
||||
if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
|
||||
if "x%~1" NEQ "x" goto usage
|
||||
|
||||
if "%DO_CLEAN%"=="false" goto fetch
|
||||
:clean
|
||||
echo.Cleaning up external libraries.
|
||||
for /D %%d in (
|
||||
bzip2-*
|
||||
db-*
|
||||
nasm-*
|
||||
openssl-*
|
||||
tcl-*
|
||||
tcltk*
|
||||
tk-*
|
||||
tix-*
|
||||
sqlite-*
|
||||
xz-*
|
||||
) do (
|
||||
echo.Removing %%d
|
||||
rmdir /s /q %%d
|
||||
)
|
||||
if "%1"=="--clean-only" (
|
||||
goto end
|
||||
if exist "%EXTERNALS_DIR%" (
|
||||
rem Sometimes this fails the first time; try it twice
|
||||
rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
|
||||
)
|
||||
|
||||
if "%DO_FETCH%"=="false" goto end
|
||||
:fetch
|
||||
rem Fetch current versions
|
||||
|
||||
svn --version > nul 2>&1
|
||||
if ERRORLEVEL 9009 (
|
||||
echo.svn.exe must be on your PATH.
|
||||
echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
|
||||
echo.command line tools option.
|
||||
popd
|
||||
exit /b 1
|
||||
if "%ORG%"=="" (set ORG=python)
|
||||
|
||||
if "%PYTHON_FOR_BUILD%"=="" (
|
||||
echo Checking for installed python...
|
||||
py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
|
||||
)
|
||||
if "%PYTHON_FOR_BUILD%"=="" (
|
||||
if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
|
||||
if NOT exist "%NUGET%" (
|
||||
echo Downloading nuget...
|
||||
rem NB: Must use single quotes around NUGET here, NOT double!
|
||||
rem Otherwise, a space in the path would break things
|
||||
powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
|
||||
)
|
||||
echo Installing Python via nuget...
|
||||
"%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
|
||||
rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
|
||||
set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
|
||||
)
|
||||
|
||||
echo.Fetching external libraries...
|
||||
|
||||
set libraries=
|
||||
set libraries=%libraries% bzip2-1.0.6
|
||||
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06
|
||||
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k
|
||||
set libraries=%libraries% sqlite-3.14.2.0
|
||||
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0
|
||||
|
|
@ -62,43 +63,48 @@ if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6
|
|||
set libraries=%libraries% xz-5.2.2
|
||||
|
||||
for %%e in (%libraries%) do (
|
||||
if exist %%e (
|
||||
if exist "%EXTERNALS_DIR%\%%e" (
|
||||
echo.%%e already exists, skipping.
|
||||
) else (
|
||||
echo.Fetching %%e...
|
||||
svn export -q %SVNROOT%%%e
|
||||
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
|
||||
)
|
||||
)
|
||||
|
||||
echo.Fetching external binaries...
|
||||
|
||||
set binaries=
|
||||
set binaries=%binaries%
|
||||
if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06
|
||||
|
||||
for %%b in (%binaries%) do (
|
||||
if exist "%EXTERNALS_DIR%\%%b" (
|
||||
echo.%%b already exists, skipping.
|
||||
) else (
|
||||
echo.Fetching %%b...
|
||||
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
|
||||
)
|
||||
)
|
||||
|
||||
echo Finished.
|
||||
goto end
|
||||
|
||||
:usage
|
||||
echo.invalid argument: %1
|
||||
echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
|
||||
echo.Valid options: -c, --clean, --clean-only, --organization, --python,
|
||||
echo.--no-tkinter, --no-openssl
|
||||
echo.
|
||||
echo.Pull all sources necessary for compiling optional extension modules
|
||||
echo.that rely on external libraries. Requires svn.exe to be on your PATH
|
||||
echo.and pulls sources from %SVNROOT%.
|
||||
echo.Pull all sources and binaries necessary for compiling optional extension
|
||||
echo.modules that rely on external libraries.
|
||||
echo.
|
||||
echo.Use the -c or --clean option to clean up all external library sources
|
||||
echo.before pulling in the current versions.
|
||||
echo.The --organization option determines which github organization to download
|
||||
echo.from, the --python option determines which Python 3.6+ interpreter to use
|
||||
echo.with PCbuild\get_external.py.
|
||||
echo.
|
||||
echo.Use the -c or --clean option to remove the entire externals directory.
|
||||
echo.
|
||||
echo.Use the --clean-only option to do the same cleaning, without pulling in
|
||||
echo.anything new.
|
||||
echo.
|
||||
echo.Only the first argument is checked, all others are ignored.
|
||||
echo.
|
||||
echo.**WARNING**: the cleaning options unconditionally remove any directory
|
||||
echo.that is a child of
|
||||
echo. %CD%
|
||||
echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
|
||||
echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
|
||||
echo.to be very destructive if you are not aware of what it is doing. Use with
|
||||
echo.caution!
|
||||
popd
|
||||
exit /b -1
|
||||
|
||||
|
||||
:end
|
||||
echo Finished.
|
||||
popd
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@ Quick Start Guide
|
|||
-----------------
|
||||
|
||||
1. Install Microsoft Visual Studio 2015, any edition.
|
||||
2. Install Subversion, and make sure 'svn.exe' is on your PATH.
|
||||
3. Run "build.bat -e" to build Python in 32-bit Release configuration.
|
||||
4. (Optional, but recommended) Run the test suite with "rt.bat -q".
|
||||
1a. Optionally install Python 3.6 or later. If not installed,
|
||||
get_externals.bat (build.bat -e) will download and use Python via
|
||||
NuGet.
|
||||
2. Run "build.bat -e" to build Python in 32-bit Release configuration.
|
||||
3. (Optional, but recommended) Run the test suite with "rt.bat -q".
|
||||
|
||||
|
||||
Building Python using Microsoft Visual C++
|
||||
|
|
@ -164,8 +166,7 @@ _bz2
|
|||
Homepage:
|
||||
http://www.bzip.org/
|
||||
_lzma
|
||||
Python wrapper for the liblzma compression library, using pre-built
|
||||
binaries of XZ Utils version 5.0.5
|
||||
Python wrapper for version 5.2.2 of the liblzma compression library
|
||||
Homepage:
|
||||
http://tukaani.org/xz/
|
||||
_ssl
|
||||
|
|
@ -236,9 +237,16 @@ order to download the relevant source files for each project before they
|
|||
can be built. However, a simple script is provided to make this as
|
||||
painless as possible, called "get_externals.bat" and located in this
|
||||
directory. This script extracts all the external sub-projects from
|
||||
http://svn.python.org/projects/external
|
||||
via Subversion (so you'll need svn.exe on your PATH) and places them
|
||||
in ..\externals (relative to this directory).
|
||||
https://github.com/python/cpython-source-deps
|
||||
and
|
||||
https://github.com/python/cpython-bin-deps
|
||||
via a Python script called "get_external.py", located in this directory.
|
||||
If Python 3.6 or later is not available via the "py.exe" launcher, the
|
||||
path or command to use for Python can be provided in the PYTHON_FOR_BUILD
|
||||
environment variable, or get_externals.bat will download the latest
|
||||
version of NuGet and use it to download the latest "pythonx86" package
|
||||
for use with get_external.py. Everything downloaded by these scripts is
|
||||
stored in ..\externals (relative to this directory).
|
||||
|
||||
It is also possible to download sources from each project's homepage,
|
||||
though you may have to change folder names or pass the names to MSBuild
|
||||
|
|
|
|||
|
|
@ -1,27 +1,107 @@
|
|||
@echo off
|
||||
setlocal
|
||||
rem Simple script to fetch source for external tools
|
||||
rem Simple script to fetch source for external libraries
|
||||
|
||||
where /Q svn
|
||||
if ERRORLEVEL 1 (
|
||||
echo.svn.exe must be on your PATH to get external tools.
|
||||
echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
|
||||
echo.command line tools option.
|
||||
popd
|
||||
exit /b 1
|
||||
set HERE=%~dp0
|
||||
if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\)
|
||||
if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer)
|
||||
if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe)
|
||||
if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
|
||||
|
||||
set DO_FETCH=true
|
||||
set DO_CLEAN=false
|
||||
|
||||
:CheckOpts
|
||||
if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
|
||||
if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
|
||||
if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
|
||||
if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
|
||||
if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
|
||||
if "x%~1" NEQ "x" goto usage
|
||||
|
||||
if "%DO_CLEAN%"=="false" goto fetch
|
||||
:clean
|
||||
echo.Cleaning up external libraries.
|
||||
if exist "%EXTERNALS_DIR%" (
|
||||
rem Sometimes this fails the first time; try it twice
|
||||
rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
|
||||
)
|
||||
|
||||
if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals"
|
||||
pushd "%~dp0..\..\externals"
|
||||
if "%DO_FETCH%"=="false" goto end
|
||||
:fetch
|
||||
|
||||
if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
|
||||
if "%ORG%"=="" (set ORG=python)
|
||||
|
||||
if not exist "windows-installer\.svn" (
|
||||
echo.Checking out installer dependencies to %CD%\windows-installer
|
||||
svn co %SVNROOT%windows-installer
|
||||
if "%PYTHON_FOR_BUILD%"=="" (
|
||||
echo Checking for installed python...
|
||||
py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
|
||||
)
|
||||
if "%PYTHON_FOR_BUILD%"=="" (
|
||||
if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
|
||||
if NOT exist "%NUGET%" (
|
||||
echo Downloading nuget...
|
||||
rem NB: Must use single quotes around NUGET here, NOT double!
|
||||
rem Otherwise, a space in the path would break things
|
||||
powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
|
||||
)
|
||||
echo Installing Python via nuget...
|
||||
"%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
|
||||
rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
|
||||
set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
|
||||
)
|
||||
|
||||
echo.Fetching external libraries...
|
||||
|
||||
set libraries=
|
||||
|
||||
for %%e in (%libraries%) do (
|
||||
if exist "%EXTERNALS_DIR%\%%e" (
|
||||
echo.%%e already exists, skipping.
|
||||
) else (
|
||||
echo.Updating installer dependencies in %CD%\windows-installer
|
||||
svn up windows-installer
|
||||
echo.Fetching %%e...
|
||||
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
|
||||
)
|
||||
)
|
||||
|
||||
popd
|
||||
echo.Fetching external tools...
|
||||
|
||||
set binaries=
|
||||
rem We always use whatever's latest in the repo for these
|
||||
set binaries=%binaries% binutils
|
||||
set binaries=%binaries% gpg
|
||||
set binaries=%binaries% htmlhelp
|
||||
set binaries=%binaries% nuget
|
||||
set binaries=%binaries% redist
|
||||
set binaries=%binaries% wix
|
||||
|
||||
for %%b in (%binaries%) do (
|
||||
if exist "%EXTERNALS_DIR%\%%b" (
|
||||
echo.%%b already exists, skipping.
|
||||
) else (
|
||||
echo.Fetching %%b...
|
||||
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
|
||||
)
|
||||
)
|
||||
|
||||
echo Finished.
|
||||
goto end
|
||||
|
||||
:usage
|
||||
echo.Valid options: -c, --clean, --clean-only, --organization, --python,
|
||||
echo.--no-tkinter, --no-openssl
|
||||
echo.
|
||||
echo.Pull all sources and binaries necessary for compiling optional extension
|
||||
echo.modules that rely on external libraries.
|
||||
echo.
|
||||
echo.The --organization option determines which github organization to download
|
||||
echo.from, the --python option determines which Python 3.6+ interpreter to use
|
||||
echo.with PCbuild\get_external.py.
|
||||
echo.
|
||||
echo.Use the -c or --clean option to remove the entire externals directory.
|
||||
echo.
|
||||
echo.Use the --clean-only option to do the same cleaning, without pulling in
|
||||
echo.anything new.
|
||||
echo.
|
||||
exit /b -1
|
||||
|
||||
:end
|
||||
|
|
|
|||
Loading…
Reference in New Issue