From 68d663cf85d1ac5eaf83482eed39c0a6f8093601 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 17 Jul 2017 11:15:48 +0200 Subject: [PATCH] [bpo-30916] Pre-build OpenSSL and Tcl/Tk for Windows (#2688) Updates ssl and tkinter projects to use pre-built externals --- Doc/make.bat | 13 +- Lib/test/test_ssl.py | 1 + .../2017-07-15-00-40-12.bpo-30916.BpCrro.rst | 1 + Modules/_ssl.c | 38 +- PCbuild/_hashlib.vcxproj | 14 +- PCbuild/_ssl.vcxproj | 19 +- PCbuild/_ssl.vcxproj.filters | 6 + PCbuild/_tkinter.vcxproj | 16 +- PCbuild/build.bat | 2 - PCbuild/find_msbuild.bat | 15 +- PCbuild/find_python.bat | 57 ++ PCbuild/get_externals.bat | 55 +- PCbuild/libeay.vcxproj | 907 ------------------ PCbuild/openssl.props | 87 +- PCbuild/openssl.vcxproj | 118 +++ PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 95 +- PCbuild/prepare_ssl.bat | 63 +- PCbuild/prepare_ssl.py | 71 +- PCbuild/prepare_tcltk.bat | 55 ++ PCbuild/pyproject.props | 2 +- PCbuild/python.props | 6 +- PCbuild/ssleay.vcxproj | 119 --- PCbuild/tcl.vcxproj | 25 +- PCbuild/tcltk.props | 5 +- PCbuild/tix.vcxproj | 18 +- PCbuild/tk.vcxproj | 21 +- Tools/msi/buildrelease.bat | 3 - Tools/msi/exe/exe.wixproj | 8 +- Tools/msi/get_externals.bat | 24 +- Tools/msi/lib/lib_files.wxs | 12 + Tools/msi/tcltk/tcltk.wixproj | 8 - Tools/msi/tcltk/tcltk_d.wxs | 1 - Tools/msi/tcltk/tcltk_files.wxs | 8 + 34 files changed, 495 insertions(+), 1400 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2017-07-15-00-40-12.bpo-30916.BpCrro.rst create mode 100644 PCbuild/find_python.bat delete mode 100644 PCbuild/libeay.vcxproj create mode 100644 PCbuild/openssl.vcxproj create mode 100644 PCbuild/prepare_tcltk.bat delete mode 100644 PCbuild/ssleay.vcxproj diff --git a/Doc/make.bat b/Doc/make.bat index d9c0ad0adec..b9e8a759c51 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -5,8 +5,19 @@ pushd %~dp0 set this=%~n0 -if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build +call ..\PCBuild\find_python.bat %PYTHON% +if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" ( + set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe + rem Cannot use %SPHINXBUILD% in the same block where we set it + if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" ( + echo Installing sphinx with %PYTHON% + "%PYTHON%" -m pip install sphinx + if errorlevel 1 exit /B + ) +) + if "%PYTHON%" EQU "" set PYTHON=py +if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build if "%1" NEQ "htmlhelp" goto :skiphhcsearch if exist "%HTMLHELP%" goto :skiphhcsearch diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index fdaf1c52046..d960d820650 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1244,6 +1244,7 @@ class ContextTests(unittest.TestCase): self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) @unittest.skipUnless(sys.platform == "win32", "Windows specific") + @unittest.skipIf(hasattr(sys, "gettotalrefcount"), "Debug build does not share environment between CRTs") def test_load_default_certs_env_windows(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) ctx.load_default_certs() diff --git a/Misc/NEWS.d/next/Windows/2017-07-15-00-40-12.bpo-30916.BpCrro.rst b/Misc/NEWS.d/next/Windows/2017-07-15-00-40-12.bpo-30916.BpCrro.rst new file mode 100644 index 00000000000..c91fd881825 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-07-15-00-40-12.bpo-30916.BpCrro.rst @@ -0,0 +1 @@ +Pre-build OpenSSL, Tcl and Tk and include the binaries in the build. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index a79a7470d20..458d2e7fd60 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -21,11 +21,13 @@ #ifdef WITH_THREAD #include "pythread.h" +/* Redefined below for Windows debug builds after important #includes */ +#define _PySSL_FIX_ERRNO #define PySSL_BEGIN_ALLOW_THREADS_S(save) \ do { if (_ssl_locks_count>0) { (save) = PyEval_SaveThread(); } } while (0) #define PySSL_END_ALLOW_THREADS_S(save) \ - do { if (_ssl_locks_count>0) { PyEval_RestoreThread(save); } } while (0) + do { if (_ssl_locks_count>0) { PyEval_RestoreThread(save); } _PySSL_FIX_ERRNO; } while (0) #define PySSL_BEGIN_ALLOW_THREADS { \ PyThreadState *_save = NULL; \ PySSL_BEGIN_ALLOW_THREADS_S(_save); @@ -96,6 +98,40 @@ struct py_ssl_library_code { int code; }; +#if defined(MS_WINDOWS) && defined(Py_DEBUG) +/* Debug builds on Windows rely on getting errno directly from OpenSSL. + * However, because it uses a different CRT, we need to transfer the + * value of errno from OpenSSL into our debug CRT. + * + * Don't be fooled - this is horribly ugly code. The only reasonable + * alternative is to do both debug and release builds of OpenSSL, which + * requires much uglier code to transform their automatically generated + * makefile. This is the lesser of all the evils. + */ + +static void _PySSLFixErrno(void) { + HMODULE ucrtbase = GetModuleHandleW(L"ucrtbase.dll"); + if (!ucrtbase) { + /* If ucrtbase.dll is not loaded but the SSL DLLs are, we likely + * have a catastrophic failure, but this function is not the + * place to raise it. */ + return; + } + + typedef int *(__stdcall *errno_func)(void); + errno_func ssl_errno = (errno_func)GetProcAddress(ucrtbase, "_errno"); + if (ssl_errno) { + errno = *ssl_errno(); + *ssl_errno() = 0; + } else { + errno = ENOTRECOVERABLE; + } +} + +#undef _PySSL_FIX_ERRNO +#define _PySSL_FIX_ERRNO _PySSLFixErrno() +#endif + /* Include generated data (error codes) */ #include "_ssl_data.h" diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index b1300cb8c9d..d6d88029d71 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -54,17 +54,15 @@ + <_ProjectFileVersion>10.0.30319.1 - - $(opensslIncludeDir);%(AdditionalIncludeDirectories) - - ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) + ws2_32.lib;%(AdditionalDependencies) @@ -78,14 +76,6 @@ {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} false - - {10615b24-73bf-4efa-93aa-236916321317} - false - - - {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} - false - diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index d75ebd656a7..aaf95a361c5 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -54,21 +54,22 @@ + <_ProjectFileVersion>10.0.30319.1 - - $(opensslIncludeDir);%(AdditionalIncludeDirectories) - - ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) + ws2_32.lib;crypt32.lib;%(AdditionalDependencies) + + _CRT_SECURE_NO_WARNINGS;$(PreprocessorDefinitions) + @@ -78,14 +79,6 @@ {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} false - - {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} - false - - - {10615b24-73bf-4efa-93aa-236916321317} - false - {86937f53-c189-40ef-8ce8-8759d8e7d480} false @@ -94,4 +87,4 @@ - + \ No newline at end of file diff --git a/PCbuild/_ssl.vcxproj.filters b/PCbuild/_ssl.vcxproj.filters index 78b1459923d..bd46b609840 100644 --- a/PCbuild/_ssl.vcxproj.filters +++ b/PCbuild/_ssl.vcxproj.filters @@ -9,5 +9,11 @@ Source Files + + Source Files + + + + \ No newline at end of file diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 67931067d3b..95e3cd50eca 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -76,19 +76,23 @@ + + <_TclTkDLL Include="$(tcltkdir)\bin\$(tclDllName)" /> + <_TclTkDLL Include="$(tcltkdir)\bin\$(tkDllName)" /> + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} false - - {b5fd6f1d-129e-4bff-9340-03606fac7283} - - - {7e85eccf-a72c-4da4-9e52-884508e80ba1} - + + + + + + \ No newline at end of file diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 713810e5b4b..3826c664adf 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options] echo. echo.Build CPython from the command line. Requires the appropriate echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). -echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is -echo.given. echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat index 1877906e00a..2b7413fbcde 100644 --- a/PCbuild/find_msbuild.bat +++ b/PCbuild/find_msbuild.bat @@ -29,6 +29,14 @@ @where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" @if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + @rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. @reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul @if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( @@ -36,13 +44,6 @@ ) @if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found -@rem VS 2015 and earlier register MSBuild separately, so we can find it. -@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul -@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( - @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") -) -@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found - @exit /b 1 diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat new file mode 100644 index 00000000000..4629c610387 --- /dev/null +++ b/PCbuild/find_python.bat @@ -0,0 +1,57 @@ +@rem +@rem Searches for python.exe and may download a private copy from nuget. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found + +:begin_search +@set PYTHON= + +@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR% +@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) + +@rem If we have Python in externals, use that one +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found + +@rem If py.exe finds a recent enough version, use that one +@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found + +@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" +@set _Py_NUGET=%NUGET% +@set _Py_NUGET_URL=%NUGET_URL% +@if "%_Py_NUGET%"=="" (set _Py_NUGET=%EXTERNALS_DIR%\nuget.exe) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if NOT exist "%_Py_NUGET%" ( + @echo Downloading nuget... + @rem NB: Must use single quotes around NUGET here, NOT double! + @rem Otherwise, a space in the path would break things + @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%' +) +@echo Installing Python via nuget... +@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work +@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found + + +@exit /b 1 + +:found +@echo Using %PYTHON% (%_Py_Python_Source%) +@set _Py_Python_Source= diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 6e466a34484..01eeb84ac0f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -4,20 +4,27 @@ rem Simple script to fetch source for external libraries if "%PCBUILD%"=="" (set PCBUILD=%~dp0) if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) -if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) set DO_FETCH=true set DO_CLEAN=false +set IncludeTkinterSrc=false +set IncludeSSLSrc=false :CheckOpts if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts -if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts +if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean + +rem Include old options for compatibility +if "%~1"=="--no-tkinter" shift & goto CheckOpts +if "%~1"=="--no-openssl" shift & goto CheckOpts + if "x%~1" NEQ "x" goto usage if "%DO_CLEAN%"=="false" goto fetch @@ -32,57 +39,41 @@ if "%DO_FETCH%"=="false" goto end :fetch if "%ORG%"=="" (set ORG=python) - -if "%PYTHON_FOR_BUILD%"=="" ( - echo Checking for installed python... - py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) -) -if "%PYTHON_FOR_BUILD%"=="" ( - if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" - if NOT exist "%NUGET%" ( - echo Downloading nuget... - rem NB: Must use single quotes around NUGET here, NOT double! - rem Otherwise, a space in the path would break things - powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' - ) - echo Installing Python via nuget... - "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" - rem Quote it here; it's not quoted later because "py -3.6" wouldn't work - set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" -) +call "%PCBUILD%find_python.bat" "%PYTHON%" echo.Fetching external libraries... set libraries= -set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k -set libraries=%libraries% sqlite-3.14.2.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.6.0 -if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 -set libraries=%libraries% xz-5.2.2 +set libraries=%libraries% bzip2-1.0.6 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.0.2k +set libraries=%libraries% sqlite-3.14.2.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.6.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 +set libraries=%libraries% xz-5.2.2 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e + %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e ) ) echo.Fetching external binaries... set binaries= -set binaries=%binaries% -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.0.2k +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.6.0 +if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( echo.%%b already exists, skipping. ) else ( echo.Fetching %%b... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b + %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b ) ) diff --git a/PCbuild/libeay.vcxproj b/PCbuild/libeay.vcxproj deleted file mode 100644 index 9662cd4b715..00000000000 --- a/PCbuild/libeay.vcxproj +++ /dev/null @@ -1,907 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0} - libeay - - - - - - - StaticLibrary - - - - - - - - <_DATEValue>#define DATE "$([System.DateTime]::Now.ToString(`ddd MMM dd HH':'mm':'ss yyyy`))" - <_CFLAGSValue>#define CFLAGS "cl /MD /Ox -W3 -Gs0 -Gy -nologo @(PreprocessorDefinitions->'-D%(Identity)',' ')" - <_PLATFORMValue Condition="$(Platform)=='Win32'">#define PLATFORM "VC-WIN32" - <_PLATFORMValue Condition="$(Platform)=='x64'">#define PLATFORM "VC-WIN64A" - - - - - - - - - - - - $(IntDir);%(AdditionalIncludeDirectories) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/PCbuild/openssl.props b/PCbuild/openssl.props index 0fc8adc9702..9ebe8a645f6 100644 --- a/PCbuild/openssl.props +++ b/PCbuild/openssl.props @@ -1,77 +1,24 @@ - + - - $(opensslDir)tmp\ - - - - - - StaticLibrary - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_PreprocessorDefinitionList>@(PreprocessorDefinitions) - - - - 4244;4267 - $(opensslDir);$(opensslIncludeDir);$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes - $(_PreprocessorDefinitionList);%(PreprocessorDefinitions) + $(opensslIncludeDir);%(AdditionalIncludeDirectories) + + $(opensslOutDir);%(AdditionalLibraryDirectories) + ws2_32.lib;libeay32.lib;ssleay32.lib;%(AdditionalDependencies) + - - - - nasm.exe -f win32 - nasm.exe -f win64 -DNEAR -Ox -g - + + <_SSLDLL Include="$(opensslOutDir)\libeay32.dll" /> + <_SSLDLL Include="$(opensslOutDir)\libeay32.pdb" /> + <_SSLDLL Include="$(opensslOutDir)\ssleay32.dll" /> + <_SSLDLL Include="$(opensslOutDir)\ssleay32.pdb" /> + + + - - - - - - - + + - + \ No newline at end of file diff --git a/PCbuild/openssl.vcxproj b/PCbuild/openssl.vcxproj new file mode 100644 index 00000000000..2d42b126818 --- /dev/null +++ b/PCbuild/openssl.vcxproj @@ -0,0 +1,118 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {B5FD6F1D-129E-4BFF-9340-03606FAC7283} + + + + + + + Makefile + 32 + 64 + x86 + amd64 + true + + + + + + + + + + + + $(opensslDir)\tmp$(Bitness)dll + $(opensslDir)\out$(Bitness)dll + ms\ntdll$(Bitness).mak + LIB_D="$(opensslOutDir.TrimEnd(`\`))" OUT_D=out$(Bitness)dll TMP_D=tmp$(Bitness)dll INC_D=inc$(Bitness) INCO_D=inc$(Bitness)\openssl + setlocal +set PATH=%PATH%;$(nasmDir);@(Perl->'%(RootDir)%(Directory)',';') +set VCINSTALLDIR=$(VCInstallDir) +cd /D "$(opensslDir.TrimEnd(`\`))" +if not exist "$(IntDir.TrimEnd('\'))" mkdir "$(IntDir.TrimEnd('\'))" +if not exist "$(OutDir.TrimEnd('\'))" mkdir "$(OutDir.TrimEnd('\'))" +if not exist "$(opensslOutDir.TrimEnd(`\`))" mkdir "$(opensslOutDir.TrimEnd(`\`))" +$(PYTHON) "@(PrepareSSL->'%(FullPath)')" "$(opensslDir.TrimEnd(`\`))" $(ArchName) +nmake -f $(MakeFile) $(NMakeOptions) headers lib +copy /y LICENSE "$(opensslOutDir)\LICENSE" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(Temp)\nmake.loc + + + <_NMakeExe Include="$(VC_ExecutablePath_x86_x86)\nmake.exe" Condition="$(VC_ExecutablePath_x86_x86) != ''" /> + + + + + + + \ No newline at end of file diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index c6b8487c0a2..8d30e0895cf 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -55,7 +55,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 280c486703d..82cfaf249d7 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -VisualStudioVersion = 15.0.26430.6 +VisualStudioVersion = 15.0.26621.2 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{553EC33E-9816-4996-A660-5D6186A0B0B3}" ProjectSection(SolutionItems) = preProject @@ -83,16 +83,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testembed", "_testembed.vc EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmultiphase.vcxproj", "{16BFE6F0-22EF-40B5-B831-7E937119EF10}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tcl", "tcl.vcxproj", "{B5FD6F1D-129E-4BFF-9340-03606FAC7283}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tix", "tix.vcxproj", "{C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tk", "tk.vcxproj", "{7E85ECCF-A72C-4DA4-9E52-884508E80BA1}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay", "libeay.vcxproj", "{E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay", "ssleay.vcxproj", "{10615B24-73BF-4EFA-93AA-236916321317}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" @@ -621,86 +611,6 @@ Global {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|Win32.Build.0 = Release|Win32 {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.ActiveCfg = Release|x64 {16BFE6F0-22EF-40B5-B831-7E937119EF10}.Release|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.ActiveCfg = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|Win32.Build.0 = Debug|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.ActiveCfg = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Debug|x64.Build.0 = Debug|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGInstrument|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.PGUpdate|x64.Build.0 = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.ActiveCfg = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|Win32.Build.0 = Release|Win32 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.ActiveCfg = Release|x64 - {B5FD6F1D-129E-4BFF-9340-03606FAC7283}.Release|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.ActiveCfg = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|Win32.Build.0 = Debug|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.ActiveCfg = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Debug|x64.Build.0 = Debug|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGInstrument|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.PGUpdate|x64.Build.0 = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.ActiveCfg = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|Win32.Build.0 = Release|Win32 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.ActiveCfg = Release|x64 - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555}.Release|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.ActiveCfg = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|Win32.Build.0 = Debug|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.ActiveCfg = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Debug|x64.Build.0 = Debug|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGInstrument|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.PGUpdate|x64.Build.0 = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.ActiveCfg = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|Win32.Build.0 = Release|Win32 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.ActiveCfg = Release|x64 - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1}.Release|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.ActiveCfg = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|Win32.Build.0 = Debug|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.ActiveCfg = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Debug|x64.Build.0 = Debug|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGInstrument|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.PGUpdate|x64.Build.0 = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.ActiveCfg = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|Win32.Build.0 = Release|Win32 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.ActiveCfg = Release|x64 - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0}.Release|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.ActiveCfg = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|Win32.Build.0 = Debug|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.ActiveCfg = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Debug|x64.Build.0 = Debug|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGInstrument|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.PGUpdate|x64.Build.0 = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.ActiveCfg = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|Win32.Build.0 = Release|Win32 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.ActiveCfg = Release|x64 - {10615B24-73BF-4EFA-93AA-236916321317}.Release|x64.Build.0 = Release|x64 {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 @@ -769,4 +679,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5C33FFD3-C8DC-4A54-B842-8BA9846BDFFE} + EndGlobalSection EndGlobal diff --git a/PCbuild/prepare_ssl.bat b/PCbuild/prepare_ssl.bat index ef8f7ad884c..1df5b8d9f52 100644 --- a/PCbuild/prepare_ssl.bat +++ b/PCbuild/prepare_ssl.bat @@ -1,12 +1,53 @@ @echo off -if not defined HOST_PYTHON ( - if "%1" EQU "Debug" ( - shift - set HOST_PYTHON=python_d.exe - if not exist python37_d.dll exit 1 - ) ELSE ( - set HOST_PYTHON=python.exe - if not exist python37.dll exit 1 - ) -) -%HOST_PYTHON% "%~dp0prepare_ssl.py" %1 +rem Downloads and build sources for libraries we depend upon + +goto Run +:Usage +echo.%~nx0 [flags and arguments] +echo. +echo.Download and build OpenSSL. This should only be performed in order to +echo.update the binaries kept online - in most cases, the files downloaded +echo.by the get_externals.bat script are sufficient for building CPython. +echo. +echo.Available flags: +echo. -h Display this help message +echo. +echo.Available arguments: +echo. --certificate (-c) The signing certificate to use for binaries. +echo. --organization The github organization to obtain sources from. +echo. +exit /b 127 + +:Run +setlocal +if "%PCBUILD%"=="" (set PCBUILD=%~dp0) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) + +set ORG_SETTING= + +:CheckOpts +if "%~1"=="-h" shift & goto Usage +if "%~1"=="--certificate" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts +if "%~1"=="-c" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts +if "%~1"=="--organization" (set ORG_SETTING=--organization "%~2") && shift && shift && goto CheckOpts + +if "%~1"=="" goto Build +echo Unrecognized option: %1 +goto Usage + +:Build +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) + +call "%PCBUILD%find_python.bat" "%PYTHON%" +if ERRORLEVEL 1 (echo Cannot locate python.exe on PATH or as PYTHON variable & exit /b 3) + +call "%PCBUILD%get_externals.bat" --openssl-src %ORG_SETTING% + +if "%PERL%" == "" where perl > "%TEMP%\perl.loc" 2> nul && set /P PERL= <"%TEMP%\perl.loc" & del "%TEMP%\perl.loc" +if "%PERL%" == "" (echo Cannot locate perl.exe on PATH or as PERL variable & exit /b 4) + +%MSBUILD% "%PCBUILD%openssl.vcxproj" /p:Configuration=Release /p:Platform=Win32 +if errorlevel 1 exit /b +%MSBUILD% "%PCBUILD%openssl.vcxproj" /p:Configuration=Release /p:Platform=x64 +if errorlevel 1 exit /b diff --git a/PCbuild/prepare_ssl.py b/PCbuild/prepare_ssl.py index 34292545192..0f3c63ee24f 100644 --- a/PCbuild/prepare_ssl.py +++ b/PCbuild/prepare_ssl.py @@ -21,6 +21,7 @@ from __future__ import print_function import os +import re import sys import subprocess from shutil import copy @@ -64,32 +65,8 @@ def find_working_perl(perls): print(" Please install ActivePerl and ensure it appears on your path") -def create_asms(makefile, tmp_d): - #create a custom makefile out of the provided one - asm_makefile = os.path.splitext(makefile)[0] + '.asm.mak' - with open(makefile) as fin, open(asm_makefile, 'w') as fout: - for line in fin: - # Keep everything up to the install target (it's convenient) - if line.startswith('install: all'): - break - fout.write(line) - asms = [] - for line in fin: - if '.asm' in line and line.strip().endswith('.pl'): - asms.append(line.split(':')[0]) - while line.strip(): - fout.write(line) - line = next(fin) - fout.write('\n') - - fout.write('asms: $(TMP_D) ') - fout.write(' '.join(asms)) - fout.write('\n') - os.system('nmake /f {} PERL=perl TMP_D={} asms'.format(asm_makefile, tmp_d)) - - def copy_includes(makefile, suffix): - dir = 'include'+suffix+'\\openssl' + dir = 'inc'+suffix+'\\openssl' try: os.makedirs(dir) except OSError: @@ -114,9 +91,29 @@ def run_configure(configure, do_script): print(do_script) os.system(do_script) +def fix_uplink(): + # uplink.c tries to find the OPENSSL_Applink function exported from the current + # executable. However, we export it from _ssl[_d].pyd instead. So we update the + # module name here before building. + with open('ms\\uplink.c', 'r', encoding='utf-8') as f1: + code = list(f1) + os.replace('ms\\uplink.c', 'ms\\uplink.c.orig') + already_patched = False + with open('ms\\uplink.c', 'w', encoding='utf-8') as f2: + for line in code: + if not already_patched: + if re.search('MODIFIED FOR CPYTHON _ssl MODULE', line): + already_patched = True + elif re.match(r'^\s+if\s*\(\(h\s*=\s*GetModuleHandle[AW]?\(NULL\)\)\s*==\s*NULL\)', line): + f2.write("/* MODIFIED FOR CPYTHON _ssl MODULE */\n") + f2.write('if ((h = GetModuleHandleW(L"_ssl.pyd")) == NULL) if ((h = GetModuleHandleW(L"_ssl_d.pyd")) == NULL)\n') + already_patched = True + f2.write(line) + if not already_patched: + print("WARN: failed to patch ms\\uplink.c") def prep(arch): - makefile_template = "ms\\nt{}.mak" + makefile_template = "ms\\ntdll{}.mak" generated_makefile = makefile_template.format('') if arch == "x86": configure = "VC-WIN32" @@ -126,13 +123,12 @@ def prep(arch): configure = "VC-WIN64A" do_script = "ms\\do_win64a" suffix = "64" - #os.environ["VSEXTCOMP_USECL"] = "MS_OPTERON" else: raise ValueError('Unrecognized platform: %s' % arch) print("Creating the makefiles...") sys.stdout.flush() - # run configure, copy includes, create asms + # run configure, copy includes, patch files run_configure(configure, do_script) makefile = makefile_template.format(suffix) try: @@ -142,9 +138,8 @@ def prep(arch): os.rename(generated_makefile, makefile) copy_includes(makefile, suffix) - print('creating asms...') - create_asms(makefile, 'tmp'+suffix) - + print('patching ms\\uplink.c...') + fix_uplink() def main(): if len(sys.argv) == 1: @@ -152,12 +147,17 @@ def main(): "sources must be supplied") sys.exit(1) - if len(sys.argv) > 2: + if len(sys.argv) == 3 and sys.argv[2] not in ('x86', 'amd64'): + print("Second argument must be x86 or amd64") + sys.exit(1) + + if len(sys.argv) > 3: print("Too many arguments supplied, all we need is the directory", - "containing OpenSSL sources") + "containing OpenSSL sources and optionally the architecture") sys.exit(1) ssl_dir = sys.argv[1] + arch = sys.argv[2] if len(sys.argv) >= 3 else None if not os.path.isdir(ssl_dir): print(ssl_dir, "is not an existing directory!") @@ -191,8 +191,11 @@ def main(): old_cwd = os.getcwd() try: os.chdir(ssl_dir) - for arch in ['amd64', 'x86']: + if arch: prep(arch) + else: + for arch in ['amd64', 'x86']: + prep(arch) finally: os.chdir(old_cwd) diff --git a/PCbuild/prepare_tcltk.bat b/PCbuild/prepare_tcltk.bat new file mode 100644 index 00000000000..7e6d43e9949 --- /dev/null +++ b/PCbuild/prepare_tcltk.bat @@ -0,0 +1,55 @@ +@echo off +rem Downloads and build sources for libraries we depend upon + +goto Run +:Usage +echo.%~nx0 [flags and arguments] +echo. +echo.Download and build Tcl/Tk. This should only be performed in order to +echo.update the binaries kept online - in most cases, the files downloaded +echo.by the get_externals.bat script are sufficient for building CPython. +echo. +echo.Available flags: +echo. -h Display this help message +echo. +echo.Available arguments: +echo. --certificate (-c) The signing certificate to use for binaries. +echo. --organization The github organization to obtain sources from. +echo. +exit /b 127 + +:Run +setlocal + +if "%PCBUILD%"=="" (set PCBUILD=%~dp0) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) + +set CERT_SETTING= +set ORG_SETTING= + +:CheckOpts +if "%~1"=="-h" shift & goto Usage +if "%~1"=="--certificate" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts +if "%~1"=="-c" (set SigningCertificate=%~2) && shift && shift & goto CheckOpts +if "%~1"=="--organization" (set ORG_SETTING=--organization "%~2") && shift && shift && goto CheckOpts + +if "%~1"=="" goto Build +echo Unrecognized option: %1 +goto Usage + +:Build +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) + +rem call "%PCBUILD%find_python.bat" "%PYTHON%" +rem if ERRORLEVEL 1 (echo Cannot locate python.exe on PATH or as PYTHON variable & exit /b 3) + +call "%PCBUILD%get_externals.bat" --tkinter-src %ORG_SETTING% + +%MSBUILD% "%PCBUILD%tcl.vcxproj" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%PCBUILD%tk.vcxproj" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%PCBUILD%tix.vcxproj" /p:Configuration=Release /p:Platform=Win32 + +%MSBUILD% "%PCBUILD%tcl.vcxproj" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%PCBUILD%tk.vcxproj" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%PCBUILD%tix.vcxproj" /p:Configuration=Release /p:Platform=x64 diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 7012170e0c7..6ab9b6533f2 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -152,7 +152,7 @@ foreach (System.Diagnostics.Process p in System.Diagnostics.Process.GetProcesses $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" diff --git a/PCbuild/python.props b/PCbuild/python.props index 563487e0c36..c81cd94fe40 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -46,8 +46,8 @@ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ $(ExternalsDir)openssl-1.0.2k\ - $(opensslDir)include32 - $(opensslDir)include64 + $(ExternalsDir)openssl-bin-1.0.2k\$(ArchName)\ + $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ @@ -72,6 +72,8 @@ --> 10.0.15063.0 10.0.15063.0 + 10.0.14393.0 + 10.0.14393.0 10.0.10586.0 10.0.10586.0 10.0.10240.0 diff --git a/PCbuild/ssleay.vcxproj b/PCbuild/ssleay.vcxproj deleted file mode 100644 index 439e3ac6745..00000000000 --- a/PCbuild/ssleay.vcxproj +++ /dev/null @@ -1,119 +0,0 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {10615B24-73BF-4EFA-93AA-236916321317} - ssleay - - - - - - - StaticLibrary - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/PCbuild/tcl.vcxproj b/PCbuild/tcl.vcxproj index 3dfd155810f..28a0ee90994 100644 --- a/PCbuild/tcl.vcxproj +++ b/PCbuild/tcl.vcxproj @@ -48,16 +48,6 @@ $(OutDir)bin\$(tclDLLName) - - - - @@ -66,28 +56,17 @@ INSTALLDIR="$(OutDir.TrimEnd(`\`))" INSTALL_DIR="$(OutDir.TrimEnd(`\`))" DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tclDir)win" nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) core shell dlls nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) install-binaries install-libraries +copy /Y ..\license.terms "$(OutDir)\tcllicense.terms" - - - - - + - - - - \ No newline at end of file diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 57bb98aeebe..b913d34b0bd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -17,9 +17,8 @@ $(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\ $(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\ $(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\ - $(ExternalsDir)tcltk\ - $(ExternalsDir)tcltk64\ - g + $(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\ + tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe diff --git a/PCbuild/tix.vcxproj b/PCbuild/tix.vcxproj index 7d6d8ca8a65..525758c1c61 100644 --- a/PCbuild/tix.vcxproj +++ b/PCbuild/tix.vcxproj @@ -37,6 +37,7 @@ {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555} tix + true @@ -49,10 +50,6 @@ $(tixDLLPath) - - - - @@ -61,17 +58,22 @@ DEBUG=0 NODEBUG=1 -c -W3 -nologo -MD -wd4028 -wd4090 setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tixDir)win" nmake /nologo -f makefile.vc MACHINE=$(TclMachine) cflags="$(CFlags)" $(DebugFlags) $(TclShortVersions) $(TixDirs) all install +copy /Y ..\license.terms "$(OutDir)\tixlicense.terms" rmdir /q/s "$(OutDir.TrimEnd(`\`))" + + + + + + + + {b5fd6f1d-129e-4bff-9340-03606fac7283} diff --git a/PCbuild/tk.vcxproj b/PCbuild/tk.vcxproj index a26318bbe78..f90e482130e 100644 --- a/PCbuild/tk.vcxproj +++ b/PCbuild/tk.vcxproj @@ -49,14 +49,6 @@ $(OutDir)bin\$(tkDLLName) - - - - @@ -65,14 +57,11 @@ TCLDIR="$(tclDir.TrimEnd(`\`))" INSTALLDIR="$(OutDir.TrimEnd(`\`))" DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tkDir)win" nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) all nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) install-binaries install-libraries +copy /Y ..\license.terms "$(OutDir)\tklicense.terms" @@ -83,15 +72,7 @@ nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index e55d81a7f70..7f9a81b33d4 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -76,9 +76,6 @@ if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & if "%SKIPBUILD%" EQU "1" goto skipdoc if "%SKIPDOC%" EQU "1" goto skipdoc -if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1 -if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1 - call "%D%..\..\doc\make.bat" htmlhelp if errorlevel 1 goto :eof :skipdoc diff --git a/Tools/msi/exe/exe.wixproj b/Tools/msi/exe/exe.wixproj index 24df0f5f7a3..50f6f882460 100644 --- a/Tools/msi/exe/exe.wixproj +++ b/Tools/msi/exe/exe.wixproj @@ -26,10 +26,10 @@ + $(opensslOutDir)LICENSE; + $(tcltkDir)tcllicense.terms; + $(tcltkDir)tklicense.terms; + $(tcltkDir)tixlicense.terms" /> <_LicenseFiles Include="@(LicenseFiles)"> $([System.IO.File]::ReadAllText(%(FullPath))) diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index aece81fbb1c..913512a778f 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -12,7 +12,7 @@ set DO_FETCH=true set DO_CLEAN=false :CheckOpts -if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts @@ -32,23 +32,7 @@ if "%DO_FETCH%"=="false" goto end if "%ORG%"=="" (set ORG=python) -if "%PYTHON_FOR_BUILD%"=="" ( - echo Checking for installed python... - py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) -) -if "%PYTHON_FOR_BUILD%"=="" ( - if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" - if NOT exist "%NUGET%" ( - echo Downloading nuget... - rem NB: Must use single quotes around NUGET here, NOT double! - rem Otherwise, a space in the path would break things - powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' - ) - echo Installing Python via nuget... - "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" - rem Quote it here; it's not quoted later because "py -3.6" wouldn't work - set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" -) +call "%PCBUILD%\find_python.bat" "%PYTHON%" echo.Fetching external libraries... @@ -59,7 +43,7 @@ for %%e in (%libraries%) do ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e + %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e ) ) @@ -79,7 +63,7 @@ for %%b in (%binaries%) do ( echo.%%b already exists, skipping. ) else ( echo.Fetching %%b... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b + %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b ) ) diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index a83f544db63..0b8a3ee89d9 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -14,6 +14,12 @@ + + + + + + @@ -30,6 +36,12 @@ + + + + + + diff --git a/Tools/msi/tcltk/tcltk.wixproj b/Tools/msi/tcltk/tcltk.wixproj index f66fc149884..fae353f5f50 100644 --- a/Tools/msi/tcltk/tcltk.wixproj +++ b/Tools/msi/tcltk/tcltk.wixproj @@ -20,14 +20,6 @@ - - $(tcltkDir) - !(bindpath.tcltk) - $(tcltkDir)bin - DLLs\ - tcltk_dlls - - $(tcltkDir) !(bindpath.tcltk) diff --git a/Tools/msi/tcltk/tcltk_d.wxs b/Tools/msi/tcltk/tcltk_d.wxs index 01d0d2439dd..391b3af5713 100644 --- a/Tools/msi/tcltk/tcltk_d.wxs +++ b/Tools/msi/tcltk/tcltk_d.wxs @@ -8,7 +8,6 @@ - diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs index 0d1b4a93a3a..11945107809 100644 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ b/Tools/msi/tcltk/tcltk_files.wxs @@ -5,9 +5,17 @@ + + + + + + + +