bpo-41282: (PEP 632) Load install schemes from sysconfig (GH-24549)

With this patch, `distutils.command.install.INSTALL_SCHEMES` are loaded from
`sysconfig._INSTALL_SCHEMES`.

The distutils module is deprecated and will be removed in 3.12 (PEP 632).
This change makes the `sysconfig._INSTALL_SCHEMES` the single point of truth
for install schemes while keeping `distutils.command.install.INSTALL_SCHEMES`
exactly the same. If we, during the transition to the sysconfig, change
something, this makes sure that it also propagates to distutils until the
module gets removed.

Moreover, as discussed [on Discourse], Linux distros need to patch
distutils/sysconfig to make sure the packages will land in proper locations.
This patch makes it easier because it leaves only one location where install
schemes are defined which is much easier to patch/adjust.

[on Discourse]: https://discuss.python.org/t/pep-632-deprecate-distutils-module/5134

The implementation is slightly different than the plan but I think it's the
easiest way how to do it and it also makes the downstream patch simple,
flexible and easy to maintain.

It's also necessary to implement this before setuptools starts bundling
the distutils module so the default install schemes stay in the standard library.

The removed code from sysconfig does not seem to have any negative effect
because, honestly, it seems that nothing actually uses the install schemes
from sysconfig at all. There were many big changes in these modules where
they were trying to include packaging in stdlib and then reverted that.
Also, the test of distutils install command does not count with the different
locations which is good evidence that the reason to have this piece of code
is no longer valid.


https://bugs.python.org/issue41282

Lib/distutils/command/install.py

@@ -3,7 +3,9 @@
 Implements the Distutils 'install' command."""
 
 import sys
+import sysconfig
 import os
+import re
 
 from distutils import log
 from distutils.core import Command
@@ -20,33 +22,45 @@ from site import USER_SITE
 
 HAS_USER_SITE = (USER_SITE is not None)
 
-WINDOWS_SCHEME = {
-    'purelib': '$base/Lib/site-packages',
-    'platlib': '$base/Lib/site-packages',
-    'headers': '$base/Include/$dist_name',
-    'scripts': '$base/Scripts',
-    'data'   : '$base',
-}
+# The keys to an installation scheme; if any new types of files are to be
+# installed, be sure to add an entry to every scheme in
+# sysconfig._INSTALL_SCHEMES, and to SCHEME_KEYS here.
+SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data')
 
-INSTALL_SCHEMES = {
-    'unix_prefix': {
-        'purelib': '$base/lib/python$py_version_short/site-packages',
-        'platlib': '$platbase/$platlibdir/python$py_version_short/site-packages',
-        'headers': '$base/include/python$py_version_short$abiflags/$dist_name',
-        'scripts': '$base/bin',
-        'data'   : '$base',
-        },
-    'unix_home': {
-        'purelib': '$base/lib/python',
-        'platlib': '$base/$platlibdir/python',
-        'headers': '$base/include/python/$dist_name',
-        'scripts': '$base/bin',
-        'data'   : '$base',
-        },
-    'nt': WINDOWS_SCHEME,
-    }
+# The following code provides backward-compatible INSTALL_SCHEMES
+# while making the sysconfig module the single point of truth.
+# This makes it easier for OS distributions where they need to
+# alter locations for packages installations in a single place.
+# Note that this module is depracated (PEP 632); all consumers
+# of this information should switch to using sysconfig directly.
+INSTALL_SCHEMES = {"unix_prefix": {}, "unix_home": {}, "nt": {}}
 
-# user site schemes
+# Copy from sysconfig._INSTALL_SCHEMES
+for key in SCHEME_KEYS:
+    sys_key = key
+    if key == "headers":
+        sys_key = "include"
+    INSTALL_SCHEMES["unix_prefix"][key] = sysconfig._INSTALL_SCHEMES["posix_prefix"][sys_key]
+    INSTALL_SCHEMES["unix_home"][key] = sysconfig._INSTALL_SCHEMES["posix_home"][sys_key]
+    INSTALL_SCHEMES["nt"][key] = sysconfig._INSTALL_SCHEMES["nt"][sys_key]
+
+# Transformation to different template format
+for main_key in INSTALL_SCHEMES:
+    for key, value in INSTALL_SCHEMES[main_key].items():
+        # Change all ocurences of {variable} to $variable
+        value = re.sub(r"\{(.+?)\}", r"$\g<1>", value)
+        value = value.replace("$installed_base", "$base")
+        value = value.replace("$py_version_nodot_plat", "$py_version_nodot")
+        if key == "headers":
+            value += "/$dist_name"
+        if sys.version_info >= (3, 9) and key == "platlib":
+            # platlibdir is available since 3.9: bpo-1294959
+            value = value.replace("/lib/", "/$platlibdir/")
+        INSTALL_SCHEMES[main_key][key] = value
+
+# The following part of INSTALL_SCHEMES has a different definition
+# than the one in sysconfig, but because both depend on the site module,
+# the outcomes should be the same.
 if HAS_USER_SITE:
     INSTALL_SCHEMES['nt_user'] = {
         'purelib': '$usersite',
@@ -65,11 +79,6 @@ if HAS_USER_SITE:
         'data'   : '$userbase',
         }
 
-# The keys to an installation scheme; if any new types of files are to be
-# installed, be sure to add an entry to every installation scheme above,
-# and to SCHEME_KEYS here.
-SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data')
-
 
 class install(Command):
 

Lib/sysconfig.py

@@ -176,12 +176,6 @@ def is_python_build(check_home=False):
 
 _PYTHON_BUILD = is_python_build(True)
 
-if _PYTHON_BUILD:
-    for scheme in ('posix_prefix', 'posix_home'):
-        _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
-        _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
-
-
 def _subst_vars(s, local_vars):
     try:
         return s.format(**local_vars)

Misc/NEWS.d/next/Library/2021-02-16-13-18-38.bpo-41282.GK9a0l.rst

@@ -0,0 +1,2 @@
+Install schemes in :mod:`distutils.command.install` are now loaded from
+:mod:`sysconfig`.