bpo-40637: Add option to disable builtin hashes (GH-20121)

Signed-off-by: Christian Heimes <christian@python.org>

Automerge-Triggered-By: @tiran

Doc/whatsnew/3.9.rst

@@ -314,6 +314,15 @@ Added a new function :func:`gc.is_finalized` to check if an object has been
 finalized by the garbage collector. (Contributed by Pablo Galindo in
 :issue:`39322`.)
 
+hashlib
+-------
+
+Builtin hash modules can now be disabled with
+``./configure --without-builtin-hashlib-hashes`` or selectively enabled with
+e.g. ``./configure --with-builtin-hashlib-hashes=sha3,blake2`` to force use
+of OpenSSL based implementation.
+(Contributed by Christian Heimes in :issue:`40479`)
+
 http
 ----
 

Misc/NEWS.d/next/Library/2020-05-15-21-57-10.bpo-40637.lb3Bnp.rst

@@ -0,0 +1,2 @@
+Builtin hash modules can now be disabled or selectively enabled with
+``configure --with-builtin-hashlib-hashes=sha3,blake1`` or ``--without-builtin-hashlib-hashes``.

configure

@@ -845,6 +845,7 @@ with_computed_gotos
 with_ensurepip
 with_openssl
 with_ssl_default_suites
+with_builtin_hashlib_hashes
 with_experimental_isolated_subinterpreters
 '
       ac_precious_vars='build_alias
@@ -1576,6 +1577,9 @@ Optional Packages:
                           leave OpenSSL's defaults untouched, STRING: use a
                           custom string, PROTOCOL_SSLv2 ignores the setting,
                           see Doc/library/ssl.rst
+  --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2
+                          builtin hash modules, md5, sha1, sha256, sha512,
+                          sha3 (with shake), blake2
   --with-experimental-isolated-subinterpreters
                           better isolate subinterpreters, experimental build
                           mode (default is no)
@@ -17493,6 +17497,44 @@ $as_echo "#define PY_SSL_DEFAULT_CIPHERS 1" >>confdefs.h
 fi
 
 
+# builtin hash modules
+default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2"
+
+$as_echo "#define PY_BUILTIN_HASHLIB_HASHES /**/" >>confdefs.h
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-builtin-hashlib-hashes" >&5
+$as_echo_n "checking for --with-builtin-hashlib-hashes... " >&6; }
+
+# Check whether --with-builtin-hashlib-hashes was given.
+if test "${with_builtin_hashlib_hashes+set}" = set; then :
+  withval=$with_builtin_hashlib_hashes;
+case "$withval" in
+    yes)
+        withval=$default_hashlib_hashes
+        ;;
+    no)
+        withval=""
+        ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
+$as_echo "$withval" >&6; }
+cat >>confdefs.h <<_ACEOF
+#define PY_BUILTIN_HASHLIB_HASHES "$withval"
+_ACEOF
+
+
+else
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $default_hashlib_hashes" >&5
+$as_echo "$default_hashlib_hashes" >&6; };
+cat >>confdefs.h <<_ACEOF
+#define PY_BUILTIN_HASHLIB_HASHES "$default_hashlib_hashes"
+_ACEOF
+
+
+fi
+
+
 # --with-experimental-isolated-subinterpreters
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-experimental-isolated-subinterpreters" >&5

configure.ac

@@ -5717,6 +5717,32 @@ AC_MSG_RESULT(python)
 AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1)
 ])
 
+# builtin hash modules
+default_hashlib_hashes="md5,sha1,sha256,sha512,sha3,blake2"
+AC_DEFINE([PY_BUILTIN_HASHLIB_HASHES], [], [enabled builtin hash modules]
+)
+AC_MSG_CHECKING(for --with-builtin-hashlib-hashes)
+AC_ARG_WITH(builtin-hashlib-hashes,
+            AS_HELP_STRING([--with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2],
+                           [builtin hash modules,
+                            md5, sha1, sha256, sha512, sha3 (with shake), blake2]),
+[
+case "$withval" in
+    yes)
+        withval=$default_hashlib_hashes
+        ;;
+    no)
+        withval=""
+        ;;
+esac
+AC_MSG_RESULT($withval)
+AC_DEFINE_UNQUOTED(PY_BUILTIN_HASHLIB_HASHES, "$withval")
+],
+[
+AC_MSG_RESULT($default_hashlib_hashes);
+AC_DEFINE_UNQUOTED(PY_BUILTIN_HASHLIB_HASHES, "$default_hashlib_hashes")
+])
+
 # --with-experimental-isolated-subinterpreters
 AH_TEMPLATE(EXPERIMENTAL_ISOLATED_SUBINTERPRETERS,
             [Better isolate subinterpreters, experimental build mode.])

pyconfig.h.in

@@ -1385,6 +1385,9 @@
 /* Define as the preferred size in bits of long digits */
 #undef PYLONG_BITS_IN_DIGIT
 
+/* enabled builtin hash modules */
+#undef PY_BUILTIN_HASHLIB_HASHES
+
 /* Define if you want to coerce the C locale to a UTF-8 based locale */
 #undef PY_COERCE_C_LOCALE
 

setup.py

@@ -327,6 +327,7 @@ class PyBuildExt(build_ext):
         self.failed = []
         self.failed_on_import = []
         self.missing = []
+        self.disabled_configure = []
         if '-j' in os.environ.get('MAKEFLAGS', ''):
             self.parallel = True
 
@@ -483,6 +484,14 @@ class PyBuildExt(build_ext):
             print_three_column([ext.name for ext in mods_disabled])
             print()
 
+        if self.disabled_configure:
+            print()
+            print("The following modules found by detect_modules() in"
+            " setup.py have not")
+            print("been built, they are *disabled* by configure:")
+            print_three_column(self.disabled_configure)
+            print()
+
         if self.failed:
             failed = self.failed[:]
             print()
@@ -2295,36 +2304,73 @@ class PyBuildExt(build_ext):
                            libraries=openssl_libs))
 
     def detect_hash_builtins(self):
-        # We always compile these even when OpenSSL is available (issue #14693).
+        # By default we always compile these even when OpenSSL is available
-        # It's harmless and the object code is tiny (40-50 KiB per module,
+        # (issue #14693). It's harmless and the object code is tiny
-        # only loaded when actually used).
+        # (40-50 KiB per module, only loaded when actually used).  Modules can
-        self.add(Extension('_sha256', ['sha256module.c'],
+        # be disabled via the --with-builtin-hashlib-hashes configure flag.
-                           extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
+        supported = {"md5", "sha1", "sha256", "sha512", "sha3", "blake2"}
-                           depends=['hashlib.h']))
-        self.add(Extension('_sha512', ['sha512module.c'],
-                           extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                           depends=['hashlib.h']))
-        self.add(Extension('_md5', ['md5module.c'],
-                           depends=['hashlib.h']))
-        self.add(Extension('_sha1', ['sha1module.c'],
-                           depends=['hashlib.h']))
 
-        blake2_deps = glob(os.path.join(self.srcdir,
+        configured = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
-                                        'Modules/_blake2/impl/*'))
+        configured = configured.strip('"').lower()
-        blake2_deps.append('hashlib.h')
+        configured = {
+            m.strip() for m in configured.split(",")
+        }
 
-        self.add(Extension('_blake2',
+        self.disabled_configure.extend(
-                           ['_blake2/blake2module.c',
+            sorted(supported.difference(configured))
-                            '_blake2/blake2b_impl.c',
+        )
-                            '_blake2/blake2s_impl.c'],
-                           depends=blake2_deps))
 
-        sha3_deps = glob(os.path.join(self.srcdir,
+        if "sha256" in configured:
-                                      'Modules/_sha3/kcp/*'))
+            self.add(Extension(
-        sha3_deps.append('hashlib.h')
+                '_sha256', ['sha256module.c'],
-        self.add(Extension('_sha3',
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                           ['_sha3/sha3module.c'],
+                depends=['hashlib.h']
-                           depends=sha3_deps))
+            ))
+
+        if "sha512" in configured:
+            self.add(Extension(
+                '_sha512', ['sha512module.c'],
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
+                depends=['hashlib.h']
+            ))
+
+        if "md5" in configured:
+            self.add(Extension(
+                '_md5', ['md5module.c'],
+                depends=['hashlib.h']
+            ))
+
+        if "sha1" in configured:
+            self.add(Extension(
+                '_sha1', ['sha1module.c'],
+                depends=['hashlib.h']
+            ))
+
+        if "blake2" in configured:
+            blake2_deps = glob(
+                os.path.join(self.srcdir, 'Modules/_blake2/impl/*')
+            )
+            blake2_deps.append('hashlib.h')
+            self.add(Extension(
+                '_blake2',
+                [
+                    '_blake2/blake2module.c',
+                    '_blake2/blake2b_impl.c',
+                    '_blake2/blake2s_impl.c'
+                ],
+                depends=blake2_deps
+            ))
+
+        if "sha3" in configured:
+            sha3_deps = glob(
+                os.path.join(self.srcdir, 'Modules/_sha3/kcp/*')
+            )
+            sha3_deps.append('hashlib.h')
+            self.add(Extension(
+                '_sha3',
+                ['_sha3/sha3module.c'],
+                depends=sha3_deps
+            ))
 
     def detect_nis(self):
         if MS_WINDOWS or CYGWIN or HOST_PLATFORM == 'qnx6':