bpo-35926: Add support for OpenSSL 1.1.1b on Windows (GH-11779)

.azure-pipelines/ci.yml

@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1b
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1b
 
   steps:
   - template: ./posix-steps.yml

Lib/test/test_asyncio/test_sslproto.py

@@ -497,8 +497,8 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin):
 
         server_context = test_utils.simple_server_sslcontext()
         client_context = test_utils.simple_client_sslcontext()
-        if sys.platform.startswith('freebsd'):
+        if sys.platform.startswith('freebsd') or sys.platform.startswith('win'):
-            # bpo-35031: Some FreeBSD buildbots fail to run this test
+            # bpo-35031: Some FreeBSD and Windows buildbots fail to run this test
             # as the eof was not being received by the server if the payload
             # size is not big enough. This behaviour only appears if the
             # client is using TLS1.3.

Lib/test/test_ssl.py

@@ -2188,7 +2188,7 @@ class ThreadedEchoServer(threading.Thread):
                     self.sock, server_side=True)
                 self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol())
                 self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol())
-            except (ConnectionResetError, BrokenPipeError) as e:
+            except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as e:
                 # We treat ConnectionResetError as though it were an
                 # SSLError - OpenSSL on Ubuntu abruptly closes the
                 # connection when asked to use an unsupported protocol.
@@ -2196,6 +2196,9 @@ class ThreadedEchoServer(threading.Thread):
                 # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL
                 # tries to send session tickets after handshake.
                 # https://github.com/openssl/openssl/issues/6342
+                #
+                # ConnectionAbortedError is raised in TLS 1.3 mode, when OpenSSL
+                # tries to send session tickets after handshake when using WinSock.
                 self.server.conn_errors.append(str(e))
                 if self.server.chatty:
                     handle_error("\n server:  bad connection attempt from " + repr(self.addr) + ":\n")
@@ -2326,7 +2329,7 @@ class ThreadedEchoServer(threading.Thread):
                             sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n"
                                              % (msg, ctype, msg.lower(), ctype))
                         self.write(msg.lower())
-                except ConnectionResetError:
+                except (ConnectionResetError, ConnectionAbortedError):
                     # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError
                     # when connection is not shut down gracefully.
                     if self.server.chatty and support.verbose:
@@ -2336,6 +2339,18 @@ class ThreadedEchoServer(threading.Thread):
                         )
                     self.close()
                     self.running = False
+                except ssl.SSLError as err:
+                    # On Windows sometimes test_pha_required_nocert receives the
+                    # PEER_DID_NOT_RETURN_A_CERTIFICATE exception
+                    # before the 'tlsv13 alert certificate required' exception.
+                    # If the server is stopped when PEER_DID_NOT_RETURN_A_CERTIFICATE
+                    # is received test_pha_required_nocert fails with ConnectionResetError
+                    # because the underlying socket is closed
+                    if 'PEER_DID_NOT_RETURN_A_CERTIFICATE' == err.reason:
+                        if self.server.chatty and support.verbose:
+                            sys.stdout.write(err.args[1])
+                        # test_pha_required_nocert is expecting this exception
+                        raise ssl.SSLError('tlsv13 alert certificate required')
                 except OSError:
                     if self.server.chatty:
                         handle_error("Test server failure:\n")

Misc/ACKS

@@ -1104,6 +1104,7 @@ Florian Mladitsch
 Doug Moen
 Jakub Molinski
 Juliette Monsel
+Paul Monson
 The Dragon De Monsyne
 Bastien Montagne
 Skip Montanaro

Misc/NEWS.d/next/Windows/2019-03-01-16-43-45.bpo-35926.mLszHo.rst

@@ -0,0 +1 @@
+Update to OpenSSL 1.1.1b for Windows.

Modules/_ssl.c

@@ -669,7 +669,7 @@ fill_and_set_sslerror(PySSLSocket *sslsock, PyObject *type, int ssl_errno,
     if (msg == NULL)
         goto fail;
 
-    init_value = Py_BuildValue("iN", ssl_errno, msg);
+    init_value = Py_BuildValue("iN", ERR_GET_REASON(ssl_errno), msg);
     if (init_value == NULL)
         goto fail;
 

PCbuild/get_externals.bat

@@ -53,7 +53,7 @@ echo.Fetching external libraries...
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries%  libffi-3.3.0-rc0-r1
-if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.0j
+if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.1b
 set libraries=%libraries%                                       sqlite-3.21.0.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0
@@ -77,7 +77,7 @@ echo.Fetching external binaries...
 
 set binaries=
 if NOT "%IncludeLibffi%"=="false"  set binaries=%binaries% libffi
-if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.0j
+if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.1b
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
 

PCbuild/openssl.props

@@ -11,7 +11,8 @@
   </ItemDefinitionGroup>
   <PropertyGroup>
     <_DLLSuffix>-1_1</_DLLSuffix>
-    <_DLLSuffix Condition="$(Platform) == 'x64'">$(_DLLSuffix)-x64</_DLLSuffix>
+    <_DLLSuffix Condition="$(Platform) == 'ARM'">$(_DLLSuffix)-arm</_DLLSuffix>
+    <_DLLSuffix Condition="$(Platform) == 'ARM64'">$(_DLLSuffix)-arm64</_DLLSuffix>
   </PropertyGroup>
   <ItemGroup>
     <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).dll" />

PCbuild/openssl.vcxproj

@@ -1,38 +1,22 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
     <ProjectConfiguration Include="Release|Win32">
       <Configuration>Release</Configuration>
       <Platform>Win32</Platform>
     </ProjectConfiguration>
-    <ProjectConfiguration Include="PGInstrument|Win32">
-      <Configuration>PGInstrument</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="PGInstrument|x64">
-      <Configuration>PGInstrument</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="PGUpdate|Win32">
-      <Configuration>PGUpdate</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="PGUpdate|x64">
-      <Configuration>PGUpdate</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Debug|x64">
-      <Configuration>Debug</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
     <ProjectConfiguration Include="Release|x64">
       <Configuration>Release</Configuration>
       <Platform>x64</Platform>
     </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{B5FD6F1D-129E-4BFF-9340-03606FAC7283}</ProjectGuid>
@@ -40,15 +24,36 @@
 
   <Import Project="python.props" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  
+
-  <PropertyGroup Label="Configuration">
+  <PropertyGroup Label="Configuration" Condition="$(Platform) == 'Win32'">
     <ConfigurationType>Makefile</ConfigurationType>
     <Bitness>32</Bitness>
-    <Bitness Condition="$(Platform) == 'x64'">64</Bitness>
     <ArchName>x86</ArchName>
-    <ArchName Condition="$(Platform) == 'x64'">amd64</ArchName>
     <OpenSSLPlatform>VC-WIN32</OpenSSLPlatform>
-    <OpenSSLPlatform Condition="$(Platform) == 'x64'">VC-WIN64A</OpenSSLPlatform>
+    <SupportSigning>true</SupportSigning>
+  </PropertyGroup>
+
+  <PropertyGroup Label="Configuration" Condition="$(Platform) == 'x64'">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <Bitness>64</Bitness>
+    <ArchName>amd64</ArchName>
+    <OpenSSLPlatform>VC-WIN64A-masm</OpenSSLPlatform>
+    <SupportSigning>true</SupportSigning>
+  </PropertyGroup>
+
+  <PropertyGroup Label="Configuration" Condition="$(Platform) == 'ARM'">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <Bitness>ARM</Bitness>
+    <ArchName>ARM</ArchName>
+    <OpenSSLPlatform>VC-WIN32-ARM</OpenSSLPlatform>
+    <SupportSigning>true</SupportSigning>
+  </PropertyGroup>
+
+  <PropertyGroup Label="Configuration" Condition="$(Platform) == 'ARM64'">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <Bitness>ARM64</Bitness>
+    <ArchName>ARM64</ArchName>
+    <OpenSSLPlatform>VC-WIN64-ARM</OpenSSLPlatform>
     <SupportSigning>true</SupportSigning>
   </PropertyGroup>
 

PCbuild/prepare_ssl.bat

@@ -42,7 +42,7 @@ if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable &
 call "%PCBUILD%\find_python.bat" "%PYTHON%"
 if ERRORLEVEL 1 (echo Cannot locate python.exe on PATH or as PYTHON variable & exit /b 3)
 
-call "%PCBUILD%\get_externals.bat" --openssl-src %ORG_SETTING%
+call "%PCBUILD%\get_externals.bat" --openssl-src --no-openssl %ORG_SETTING%
 
 if "%PERL%" == "" where perl > "%TEMP%\perl.loc" 2> nul && set /P PERL= <"%TEMP%\perl.loc" & del "%TEMP%\perl.loc"
 if "%PERL%" == "" (echo Cannot locate perl.exe on PATH or as PERL variable & exit /b 4)
@@ -51,4 +51,8 @@ if "%PERL%" == "" (echo Cannot locate perl.exe on PATH or as PERL variable & exi
 if errorlevel 1 exit /b
 %MSBUILD% "%PCBUILD%\openssl.vcxproj" /p:Configuration=Release /p:Platform=x64
 if errorlevel 1 exit /b
+%MSBUILD% "%PCBUILD%\openssl.vcxproj" /p:Configuration=Release /p:Platform=ARM
+if errorlevel 1 exit /b
+%MSBUILD% "%PCBUILD%\openssl.vcxproj" /p:Configuration=Release /p:Platform=ARM64
+if errorlevel 1 exit /b
 

PCbuild/python.props

@@ -26,6 +26,7 @@
     -->
     <ArchName Condition="'$(ArchName)' == '' and $(Platform) == 'x64'">amd64</ArchName>
     <ArchName Condition="'$(ArchName)' == '' and $(Platform) == 'ARM'">arm32</ArchName>
+    <ArchName Condition="'$(ArchName)' == '' and $(Platform) == 'ARM64'">arm64</ArchName>
     <ArchName Condition="'$(ArchName)' == ''">win32</ArchName>
     
     <!-- Root directory of the repository -->
@@ -56,8 +57,8 @@
     <libffiDir>$(ExternalsDir)libffi\</libffiDir>
     <libffiOutDir>$(ExternalsDir)libffi\$(ArchName)\</libffiOutDir>
     <libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir>
-    <opensslDir>$(ExternalsDir)openssl-1.1.0j\</opensslDir>
+    <opensslDir>$(ExternalsDir)openssl-1.1.1b\</opensslDir>
-    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.0j\$(ArchName)\</opensslOutDir>
+    <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1b\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>
     <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir>
     <zlibDir>$(ExternalsDir)\zlib-1.2.11\</zlibDir>

PCbuild/readme.txt

@@ -165,7 +165,7 @@ _lzma
     Homepage:
         http://tukaani.org/xz/
 _ssl
-    Python wrapper for version 1.1.0h of the OpenSSL secure sockets
+    Python wrapper for version 1.1.1b of the OpenSSL secure sockets
     library, which is downloaded from our binaries repository at
     https://github.com/python/cpython-bin-deps.