From e12fa657447c91356b3c52dac9b1e4da937bd26c Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 7 Jul 2014 13:39:59 -0500 Subject: [PATCH] Issue #21907: Improved the batch scripts provided for building Python. The user-facing scripts in PCbuild have been updated to be easier to use and the buildbot scripts in Tools\buildbot have been updated to use the user-facing scripts in PCbuild wherever possible. --- Misc/NEWS | 2 + PCbuild/build.bat | 27 ++++++-- PCbuild/build_pgo.bat | 12 ++-- PCbuild/env.bat | 6 +- PCbuild/get_externals.bat | 100 +++++++++++++++++++++++++++++ PCbuild/readme.txt | 57 +++++++++++----- Tools/buildbot/build-amd64.bat | 6 +- Tools/buildbot/build.bat | 18 ++++-- Tools/buildbot/buildmsi.bat | 15 ++--- Tools/buildbot/clean-amd64.bat | 10 +-- Tools/buildbot/clean.bat | 30 ++++++--- Tools/buildbot/external-amd64.bat | 8 +-- Tools/buildbot/external-common.bat | 47 -------------- Tools/buildbot/external.bat | 7 +- Tools/buildbot/test-amd64.bat | 6 +- Tools/buildbot/test.bat | 6 +- Tools/scripts/run_tests.py | 6 +- 17 files changed, 230 insertions(+), 133 deletions(-) create mode 100644 PCbuild/get_externals.bat delete mode 100644 Tools/buildbot/external-common.bat diff --git a/Misc/NEWS b/Misc/NEWS index 1b760e77fb7..af1fe3a57df 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -746,6 +746,8 @@ Tools/Demos Windows ------- +- Issue #21907: Improved the batch scripts provided for building Python. + - Issue #21671, CVE-2014-0224: The bundled version of OpenSSL has been updated to 1.0.1h. diff --git a/PCbuild/build.bat b/PCbuild/build.bat index fa0d95b0cc0..cde88c5a7ad 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -1,19 +1,34 @@ @echo off -rem A batch program to build or rebuild a particular configuration. +rem A batch program to build or rebuild a particular configuration, rem just for convenience. +rem Arguments: +rem -c Set the configuration (default: Release) +rem -p Set the platform (x64 or Win32, default: Win32) +rem -r Target Rebuild instead of Build +rem -d Set the configuration to Debug +rem -e Pull in external libraries using get_externals.bat + setlocal set platf=Win32 set conf=Release -set target=build +set target=Build set dir=%~dp0 :CheckOpts if "%1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts if "%1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts -if "%1"=="-r" (set target=rebuild) & shift & goto CheckOpts +if "%1"=="-r" (set target=Rebuild) & shift & goto CheckOpts if "%1"=="-d" (set conf=Debug) & shift & goto CheckOpts +if "%1"=="-e" call "%dir%get_externals.bat" & shift & goto CheckOpts -set cmd=msbuild /p:useenv=true %dir%pcbuild.sln /t:%target% /p:Configuration=%conf% /p:Platform=%platf% -echo %cmd% -%cmd% +if "%platf%"=="x64" (set vs_platf=x86_amd64) + +rem Setup the environment +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %vs_platf% + +rem Call on MSBuild to do the work, echo the command. +rem Passing %1-9 is not the preferred option, but argument parsing in +rem batch is, shall we say, "lackluster" +echo on +msbuild "%dir%pcbuild.sln" /t:%target% /p:Configuration=%conf% /p:Platform=%platf% %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/build_pgo.bat b/PCbuild/build_pgo.bat index 0c0a473b2c4..6ead19a3de5 100644 --- a/PCbuild/build_pgo.bat +++ b/PCbuild/build_pgo.bat @@ -9,12 +9,12 @@ setlocal set platf=Win32 rem use the performance testsuite. This is quick and simple -set job1=..\tools\pybench\pybench.py -n 1 -C 1 --with-gc -set path1=..\tools\pybench +set job1=%~dp0..\tools\pybench\pybench.py -n 1 -C 1 --with-gc +set path1=%~dp0..\tools\pybench rem or the whole testsuite for more thorough testing -set job2=..\lib\test\regrtest.py -set path2=..\lib +set job2=%~dp0..\lib\test\regrtest.py +set path2=%~dp0..\lib set job=%job1% set clrpath=%path1% @@ -31,9 +31,9 @@ rem build the instrumented version call build -p %platf% -c PGInstrument rem remove .pyc files, .pgc files and execute the job -%PGI%\python.exe rmpyc.py %clrpath% +%PGI%\python.exe rmpyc.py "%clrpath%" del %PGI%\*.pgc -%PGI%\python.exe %job% +%PGI%\python.exe "%job%" rem finally build the optimized version if exist %PGO% del /s /q %PGO% diff --git a/PCbuild/env.bat b/PCbuild/env.bat index 08dc8ef5253..f0268a9135c 100644 --- a/PCbuild/env.bat +++ b/PCbuild/env.bat @@ -1,9 +1,5 @@ @echo off -set VS10=%ProgramFiles(x86)%\Microsoft Visual Studio 10.0 -IF EXIST "%VS10%" GOTO ok -set VS10=%ProgramFiles%\Microsoft Visual Studio 10.0 -:ok echo Build environments: x86, ia64, amd64, x86_amd64, x86_ia64 echo. -call "%VS10%\VC\vcvarsall.bat" %1 +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %1 diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat new file mode 100644 index 00000000000..f472448121a --- /dev/null +++ b/PCbuild/get_externals.bat @@ -0,0 +1,100 @@ +@echo off +setlocal +rem Simple script to fetch source for external libraries + +pushd "%~dp0..\.." + +if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ + +rem Optionally clean up first. Be warned that this can be very destructive! +if not "%1"=="" ( + for %%c in (-c --clean --clean-only) do ( + if "%1"=="%%c" goto clean + ) + goto usage +) +goto fetch + +:clean +echo.Cleaning up external libraries. +for /D %%d in ( + bzip2-* + db-* + openssl-* + tcl-* + tcltk* + tk-* + tix-* + sqlite-* + xz-* + ) do ( + echo.Removing %%d + rmdir /s /q %%d +) +if "%1"=="--clean-only" ( + goto end +) + +:fetch +rem Fetch current versions + +svn --version > nul 2>&1 +if ERRORLEVEL 9009 ( + echo.svn.exe must be on your PATH. + echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the + echo.command line tools option. + popd + exit /b 1 +) + +echo.Fetching external libraries... + +for %%e in ( + bzip2-1.0.6 + openssl-1.0.1h + tcl-8.6.1.0 + tk-8.6.1.0 + tix-8.4.3.4 + sqlite-3.8.3.1 + xz-5.0.5 + ) do ( + if exist %%e ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + svn export %SVNROOT%%%e + ) +) + +goto end + +:usage +echo.invalid argument: %1 +echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo. +echo.Pull all sources necessary for compiling optional extension modules +echo.that rely on external libraries. Requires svn.exe to be on your PATH +echo.and pulls sources from %SVNROOT%. +echo. +echo.Use the -c or --clean option to clean up all external library sources +echo.before pulling in the current versions. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +echo.Only the first argument is checked, all others are ignored. +echo. +echo.**WARNING**: the cleaning options unconditionally remove any directory +echo.that is a child of +echo. %CD% +echo.and matches wildcard patterns beginning with bzip2-, db-, openssl-, tcl-, +echo.tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential to be +echo.very destructive if you are not aware of what it is doing. Use with +echo.caution! +popd +exit /b -1 + + +:end +echo Finished. +popd diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c300e09a14b..581ff833063 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -1,3 +1,13 @@ +Quick Start Guide +----------------- + +1. Install Microsoft Visual C++ 2010 SP1, any edition. +2. Install Subversion, and make sure 'svn.exe' is on your PATH. +3. Install NASM, and make sure 'nasm.exe' is on your PATH. +4. Run "build.bat -e" to build Python in 32-bit Release configuration. +5. (Optional, but recommended) Run the test suite with "rt.bat -q". + + Building Python using Microsoft Visual C++ ------------------------------------------ @@ -24,8 +34,8 @@ All you need to do to build is open the solution "pcbuild.sln" in Visual Studio, select the desired combination of configuration and platform, then build with "Build Solution" or the F7 keyboard shortcut. You can also build from the command line using the "build.bat" script in this -directory. The solution is configured to build the projects in the -correct order. +directory; see below for details. The solution is configured to build +the projects in the correct order. The solution currently supports two platforms. The Win32 platform is used to build standard x86-compatible 32-bit binaries, output into this @@ -56,6 +66,26 @@ Release settings, though without PGO. +Building Python using the build.bat script +---------------------------------------------- + +In this directory you can find build.bat, a script designed to make +building Python on Windows simpler. The only absolute requirement for +using this script is for the VS100COMNTOOLS environment variable to be +properly set, which should be done by Microsoft Visual C++ 2010 +installation. + +By default, build.bat will build Python in Release configuration for +the 32-bit Win32 platform. It accepts several arguments to change +this behavior: + + -c Set the configuration (see above) + -d Shortcut for "-c Debug" + -p Set the platform to build for ("Win32" or "x64") + -r Rebuild instead of just building + -e Use get_externals.bat to fetch external sources + + Legacy support -------------- @@ -227,25 +257,18 @@ Getting External Sources The last category of sub-projects listed above wrap external projects Python doesn't control, and as such a little more work is required in order to download the relevant source files for each project before they -can be built. The buildbots must ensure that all libraries are present -before building, so the easiest approach is to run either external.bat -or external-amd64.bat (depending on platform) in the ..\Tools\buildbot -directory from ..\, i.e.: - - C:\python\cpython\PCbuild>cd .. - C:\python\cpython>Tools\buildbot\external.bat - -This extracts all the external sub-projects from +can be built. However, a simple script is provided to make this as +painless as possible, called "get_externals.bat" and located in this +directory. This script extracts all the external sub-projects from http://svn.python.org/projects/external -via Subversion (so you'll need an svn.exe on your PATH) and places them +via Subversion (so you'll need svn.exe on your PATH) and places them in ..\.. (relative to this directory). It is also possible to download sources from each project's homepage, -though you may have to change the names of some folders in order to make -things work. For instance, if you were to download a version 5.0.7 of -XZ Utils, you would need to extract the archive into ..\..\xz-5.0.5 -anyway, since that is where the solution is set to look for xz. The -same is true for all other external projects. +though you may have to change folder names or pass the names to MSBuild +as the values of certain properties in order for the build solution to +find them. This is an advanced topic and not necessarily fully +supported. Building for AMD64 diff --git a/Tools/buildbot/build-amd64.bat b/Tools/buildbot/build-amd64.bat index 493e74de5a3..115e111f76d 100644 --- a/Tools/buildbot/build-amd64.bat +++ b/Tools/buildbot/build-amd64.bat @@ -1,6 +1,2 @@ @rem Used by the buildbot "compile" step. -cmd /c Tools\buildbot\external-amd64.bat -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 -cmd /c Tools\buildbot\clean-amd64.bat - -msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 +call "%~dp0build.bat" -p x64 %* diff --git a/Tools/buildbot/build.bat b/Tools/buildbot/build.bat index be79b10726d..d37ec7b31c4 100644 --- a/Tools/buildbot/build.bat +++ b/Tools/buildbot/build.bat @@ -1,7 +1,17 @@ @rem Used by the buildbot "compile" step. -cmd /c Tools\buildbot\external.bat -call "%VS100COMNTOOLS%vsvars32.bat" -cmd /c Tools\buildbot\clean.bat -msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 +@rem Clean up +call "%~dp0clean.bat" +@rem If you need the buildbots to start fresh (such as when upgrading to +@rem a new version of an external library, especially Tcl/Tk): +@rem 1) uncomment the following line: + +@rem call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only + +@rem 2) commit and push +@rem 3) wait for all Windows bots to start a build with that changeset +@rem 4) re-comment, commit and push again + +@rem Do the build +call "%~dp0..\..\PCbuild\build.bat" -e -d %* diff --git a/Tools/buildbot/buildmsi.bat b/Tools/buildbot/buildmsi.bat index ae93e672956..f414c22e6a8 100644 --- a/Tools/buildbot/buildmsi.bat +++ b/Tools/buildbot/buildmsi.bat @@ -1,21 +1,20 @@ @rem Used by the buildbot "buildmsi" step. +setlocal -cmd /c Tools\buildbot\external.bat +set cwd=%CD% @rem build release versions of things -call "%VS100COMNTOOLS%vsvars32.bat" - -@rem build Python -msbuild /p:useenv=true PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 +call "%~dp0build.bat" -c Release @rem build the documentation -bash.exe -c 'cd Doc;make PYTHON=python2.5 update htmlhelp' -"%ProgramFiles%\HTML Help Workshop\hhc.exe" Doc\build\htmlhelp\python26a3.hhp +call "%~dp0..\..\Doc\make.bat" htmlhelp @rem build the MSI file -cd PC +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" x86 +cd "%~dp0..\..\PC" nmake /f icons.mak cd ..\Tools\msi del *.msi nmake /f msisupport.mak %HOST_PYTHON% msi.py +cd "%cwd%" diff --git a/Tools/buildbot/clean-amd64.bat b/Tools/buildbot/clean-amd64.bat index 24660af012a..4fcd6d435d0 100644 --- a/Tools/buildbot/clean-amd64.bat +++ b/Tools/buildbot/clean-amd64.bat @@ -1,10 +1,2 @@ @rem Used by the buildbot "clean" step. -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 -@echo Deleting .pyc/.pyo files ... -del /s Lib\*.pyc Lib\*.pyo -@echo Deleting test leftovers ... -rmdir /s /q build -cd PCbuild -msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 -msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 -cd .. +call "%~dp0clean.bat" diff --git a/Tools/buildbot/clean.bat b/Tools/buildbot/clean.bat index 218facc3b17..1684e8ca9b8 100644 --- a/Tools/buildbot/clean.bat +++ b/Tools/buildbot/clean.bat @@ -1,8 +1,22 @@ -@rem Used by the buildbot "clean" step. -call "%VS100COMNTOOLS%vsvars32.bat" -@echo Deleting test leftovers ... -rmdir /s /q build -cd PCbuild -msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x86 -msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x86 -cd .. +@echo off +rem Used by the buildbot "clean" step. + +setlocal +set root=%~dp0..\.. +set pcbuild=%root%\PCbuild + +echo.Attempting to kill Pythons... +for %%k in (kill_python.exe + kill_python_d.exe + amd64\kill_python.exe + amd64\kill_python_d.exe + ) do ( + if exist "%pcbuild%\%%k" ( + echo.Calling %pcbuild%\%%k... + "%pcbuild%\%%k" + ) +) + +echo Purging all non-tracked files with `hg purge` +echo on +hg -R "%root%" --config extensions.purge= purge --all -X "%root%\Lib\test\data" diff --git a/Tools/buildbot/external-amd64.bat b/Tools/buildbot/external-amd64.bat index f8597706b11..7a570d92ba1 100644 --- a/Tools/buildbot/external-amd64.bat +++ b/Tools/buildbot/external-amd64.bat @@ -1,6 +1,2 @@ -@rem Fetches (and builds if necessary) external dependencies - -@rem Assume we start inside the Python source directory -call "Tools\buildbot\external-common.bat" - - +@echo Please use PCbuild\get_externals.bat instead. +@"%~dp0..\..\PCbuild\get_externals.bat" %* diff --git a/Tools/buildbot/external-common.bat b/Tools/buildbot/external-common.bat deleted file mode 100644 index a58959e0ac8..00000000000 --- a/Tools/buildbot/external-common.bat +++ /dev/null @@ -1,47 +0,0 @@ -@rem Common file shared between external.bat and external-amd64.bat. Responsible for -@rem fetching external components into the root\.. buildbot directories. - -cd .. -@rem XXX: If you need to force the buildbots to start from a fresh environment, uncomment -@rem the following, check it in, then check it out, comment it out, then check it back in. -@rem if exist bzip2-1.0.6 rd /s/q bzip2-1.0.6 -@rem if exist tcltk rd /s/q tcltk -@rem if exist tcltk64 rd /s/q tcltk64 -@rem if exist tcl-8.6.1.0 rd /s/q tcl-8.6.1.0 -@rem if exist tk-8.6.1.0 rd /s/q tk-8.6.1.0 -@rem if exist tix-8.4.3.4 rd /s/q tix-8.4.3.4 -@rem if exist db-4.4.20 rd /s/q db-4.4.20 -@rem if exist openssl-1.0.1h rd /s/q openssl-1.0.1h -@rem if exist sqlite-3.7.12 rd /s/q sqlite-3.7.12 - -@rem bzip -if not exist bzip2-1.0.6 ( - rd /s/q bzip2-1.0.5 - svn export http://svn.python.org/projects/external/bzip2-1.0.6 -) - -@rem OpenSSL -if not exist openssl-1.0.1h ( - rd /s/q openssl-1.0.1g - svn export http://svn.python.org/projects/external/openssl-1.0.1h -) - -@rem tcl/tk/tix -if not exist tcl-8.6.1.0 ( - rd /s/q tcltk tcltk64 tcl-8.5.11.0 tk-8.5.11.0 tix-8.4.3.3 - svn export http://svn.python.org/projects/external/tcl-8.6.1.0 -) -if not exist tk-8.6.1.0 svn export http://svn.python.org/projects/external/tk-8.6.1.0 -if not exist tix-8.4.3.4 svn export http://svn.python.org/projects/external/tix-8.4.3.4 - -@rem sqlite3 -if not exist sqlite-3.8.3.1 ( - rd /s/q sqlite-source-3.8.1 - svn export http://svn.python.org/projects/external/sqlite-3.8.3.1 -) - -@rem lzma -if not exist xz-5.0.5 ( - rd /s/q xz-5.0.3 - svn export http://svn.python.org/projects/external/xz-5.0.5 -) diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat index 22dfdc50ecf..7a570d92ba1 100644 --- a/Tools/buildbot/external.bat +++ b/Tools/buildbot/external.bat @@ -1,5 +1,2 @@ -@rem Fetches (and builds if necessary) external dependencies - -@rem Assume we start inside the Python source directory -call "Tools\buildbot\external-common.bat" - +@echo Please use PCbuild\get_externals.bat instead. +@"%~dp0..\..\PCbuild\get_externals.bat" %* diff --git a/Tools/buildbot/test-amd64.bat b/Tools/buildbot/test-amd64.bat index 11bfcbbca25..b6932771b1f 100644 --- a/Tools/buildbot/test-amd64.bat +++ b/Tools/buildbot/test-amd64.bat @@ -1,6 +1,6 @@ @rem Used by the buildbot "test" step. rem The following line should be removed before #20035 is closed -set TCL_LIBRARY=%CD%\..\tcltk64\lib\tcl8.6 -cd PCbuild -call rt.bat -d -q -x64 -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9 +set TCL_LIBRARY=%~dp0..\..\..\tcltk64\lib\tcl8.6 + +"%~dp0..\..\PCbuild\amd64\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %* diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 6b98c79015e..ec6eaff4f06 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -1,6 +1,6 @@ @rem Used by the buildbot "test" step. rem The following line should be removed before #20035 is closed -set TCL_LIBRARY=%CD%\..\tcltk\lib\tcl8.6 -cd PCbuild -call rt.bat -d -q -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9 +set TCL_LIBRARY=%~dp0..\..\..\tcltk\lib\tcl8.6 + +"%~dp0..\..\PCbuild\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %* diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py index 490a37e175c..b451a75e79b 100644 --- a/Tools/scripts/run_tests.py +++ b/Tools/scripts/run_tests.py @@ -48,7 +48,11 @@ def main(regrtest_args): args.extend(['-u', 'all,-largefile,-audio,-gui']) args.extend(regrtest_args) print(' '.join(args)) - os.execv(sys.executable, args) + if sys.platform == 'win32': + from subprocess import call + call(args) + else: + os.execv(sys.executable, args) if __name__ == '__main__':