bpo-32549: Compile OpenSSL 1.1.0 on Travis CI (#5180)

Use an improved version of multissl test helper to compile a local copy
of OpenSSL 1.1.0g.

Signed-off-by: Christian Heimes <christian@python.org>

.travis.yml

@@ -7,6 +7,19 @@ group: beta
 cache:
     - pip
     - ccache
+    - directories:
+        - $HOME/multissl
+
+env:
+  global:
+    - OPENSSL=1.1.0g
+    - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
+    - PATH="${OPENSSL_DIR}/bin:$PATH"
+    - CFLAGS="-I${OPENSSL_DIR}/include"
+    - LDFLAGS="-L${OPENSSL_DIR}/lib"
+    # Set rpath with env var instead of -Wl,-rpath linker flag
+    # OpenSSL ignores LDFLAGS when linking bin/openssl
+    - LD_RUN_PATH="${OPENSSL_DIR}/lib"
 
 branches:
   only:
@@ -48,6 +61,10 @@ matrix:
               echo "Only docs were updated, stopping build process."
               exit
             fi
+            python3 Tools/ssl/multissltests.py  --steps=library \
+                --base-directory ${HOME}/multissl \
+                --openssl ${OPENSSL} >/dev/null
+            openssl version
             ./configure
             make -s -j4
             # Need a venv that can parse covered code.
@@ -71,6 +88,13 @@ before_script:
         echo "Only docs were updated, stopping build process."
         exit
       fi
+      if [ "${TESTING}" != "docs" ]; then
+        # clang complains about unused-parameter a lot, redirect stderr
+        python3 Tools/ssl/multissltests.py --steps=library \
+            --base-directory ${HOME}/multissl \
+            --openssl ${OPENSSL} >/dev/null 2>&1
+      fi
+      openssl version
       ./configure --with-pydebug
       make -j4
       make -j4 regen-all clinic

Makefile.pre.in

@@ -1086,7 +1086,7 @@ quicktest:	@DEF_MAKE_RULE@ platform
 # SSL tests
 .PHONY: multisslcompile multissltest
 multisslcompile: build_all
-	$(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py --compile-only
+	$(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py --steps=modules
 
 multissltest: build_all
 	$(RUNSHARED) ./$(BUILDPYTHON) Tools/ssl/multissltests.py

Misc/NEWS.d/next/Tests/2018-01-14-11-40-22.bpo-32549.fLwbVA.rst

@@ -0,0 +1 @@
+On Travis CI, Python now Compiles and uses a local copy of OpenSSL 1.1.0g for testing.

Tools/ssl/multissltests.py

@@ -41,8 +41,8 @@ import tarfile
 log = logging.getLogger("multissl")
 
 OPENSSL_OLD_VERSIONS = [
-     "0.9.8zh",
+     # "0.9.8zh",
-     "1.0.1u",
+     # "1.0.1u",
 ]
 
 OPENSSL_RECENT_VERSIONS = [
@@ -52,8 +52,8 @@ OPENSSL_RECENT_VERSIONS = [
 ]
 
 LIBRESSL_OLD_VERSIONS = [
-    "2.3.10",
+    # "2.3.10",
-    "2.4.5",
+    # "2.4.5",
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
@@ -62,8 +62,10 @@ LIBRESSL_RECENT_VERSIONS = [
 ]
 
 # store files in ../multissl
-HERE = os.path.abspath(os.getcwd())
+HERE = os.path.dirname(os.path.abspath(__file__))
-MULTISSL_DIR = os.path.abspath(os.path.join(HERE, '..', 'multissl'))
+PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..'))
+MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl'))
+
 
 parser = argparse.ArgumentParser(
     prog='multissl',
@@ -118,9 +120,14 @@ parser.add_argument(
     help="Disable network tests."
 )
 parser.add_argument(
-    '--compile-only',
+    '--steps',
-    action='store_true',
+    choices=['library', 'modules', 'tests'],
-    help="Don't run tests, only compile _ssl.c and _hashopenssl.c."
+    default='tests',
+    help=(
+        "Which steps to perform. 'library' downloads and compiles OpenSSL "
+        "or LibreSSL. 'module' also compiles Python modules. 'tests' builds "
+        "all and runs the test suite."
+    )
 )
 
 
@@ -129,21 +136,21 @@ class AbstractBuilder(object):
     url_template = None
     src_template = None
     build_template = None
+    install_target = 'install'
 
     module_files = ("Modules/_ssl.c",
                     "Modules/_hashopenssl.c")
     module_libs = ("_ssl", "_hashlib")
 
-    def __init__(self, version, compile_args=(),
+    def __init__(self, version, args):
-                 basedir=MULTISSL_DIR):
         self.version = version
-        self.compile_args = compile_args
+        self.args = args
         # installation directory
         self.install_dir = os.path.join(
-            os.path.join(basedir, self.library.lower()), version
+            os.path.join(args.base_directory, self.library.lower()), version
         )
         # source file
-        self.src_dir = os.path.join(basedir, 'src')
+        self.src_dir = os.path.join(args.base_directory, 'src')
         self.src_file = os.path.join(
             self.src_dir, self.src_template.format(version))
         # build directory (removed after install)
@@ -252,13 +259,15 @@ class AbstractBuilder(object):
         log.info("Running build in {}".format(self.build_dir))
         cwd = self.build_dir
         cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
-        cmd.extend(self.compile_args)
         self._subprocess_call(cmd, cwd=cwd)
         # Old OpenSSL versions do not support parallel builds.
         self._subprocess_call(["make", "-j1"], cwd=cwd)
 
     def _make_install(self, remove=True):
-        self._subprocess_call(["make", "-j1", "install"], cwd=self.build_dir)
+        self._subprocess_call(
+            ["make", "-j1", self.install_target],
+            cwd=self.build_dir
+        )
         if remove:
             shutil.rmtree(self.build_dir)
 
@@ -330,6 +339,8 @@ class BuildOpenSSL(AbstractBuilder):
     url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
     src_template = "openssl-{}.tar.gz"
     build_template = "openssl-{}"
+    # only install software, skip docs
+    install_target = 'install_sw'
 
 
 class BuildLibreSSL(AbstractBuilder):
@@ -368,57 +379,63 @@ def main():
 
     start = datetime.now()
 
-    for name in ['python', 'setup.py', 'Modules/_ssl.c']:
+    if args.steps in {'modules', 'tests'}:
-        if not os.path.isfile(name):
+        for name in ['setup.py', 'Modules/_ssl.c']:
+            if not os.path.isfile(os.path.join(PYTHONROOT, name)):
+                parser.error(
+                    "Must be executed from CPython build dir"
+                )
+        if not os.path.samefile('python', sys.executable):
             parser.error(
-                "Must be executed from CPython build dir"
+                "Must be executed with ./python from CPython build dir"
             )
-    if not os.path.samefile('python', sys.executable):
+        # check for configure and run make
-        parser.error(
+        configure_make()
-            "Must be executed with ./python from CPython build dir"
-        )
-
-    # check for configure and run make
-    configure_make()
 
     # download and register builder
     builds = []
 
     for version in args.openssl:
-        build = BuildOpenSSL(version)
+        build = BuildOpenSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
     for version in args.libressl:
-        build = BuildLibreSSL(version)
+        build = BuildLibreSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
-    for build in builds:
+    if args.steps in {'modules', 'tests'}:
-        try:
+        for build in builds:
-            build.recompile_pymods()
+            try:
-            build.check_pyssl()
+                build.recompile_pymods()
-            if not args.compile_only:
+                build.check_pyssl()
-                build.run_python_tests(
+                if args.steps == 'tests':
-                    tests=args.tests,
+                    build.run_python_tests(
-                    network=args.network,
+                        tests=args.tests,
-                )
+                        network=args.network,
-        except Exception as e:
+                    )
-            log.exception("%s failed", build)
+            except Exception as e:
-            print("{} failed: {}".format(build, e), file=sys.stderr)
+                log.exception("%s failed", build)
-            sys.exit(2)
+                print("{} failed: {}".format(build, e), file=sys.stderr)
+                sys.exit(2)
 
-    print("\n{} finished in {}".format(
+    log.info("\n{} finished in {}".format(
-        "Tests" if not args.compile_only else "Builds",
+            args.steps.capitalize(),
-        datetime.now() - start
+            datetime.now() - start
-    ))
+        ))
     print('Python: ', sys.version)
-    if args.compile_only:
+    if args.steps == 'tests':
-        print('Build only')
+        if args.tests:
-    elif args.tests:
+            print('Executed Tests:', ' '.join(args.tests))
-        print('Executed Tests:', ' '.join(args.tests))
+        else:
-    else:
+            print('Executed all SSL tests.')
-        print('Executed all SSL tests.')
 
     print('OpenSSL / LibreSSL versions:')
     for build in builds: