bpo-29585: optimize site.py startup time (GH-136)
Avoid importing `sysconfig` from `site` by copying minimum code. Python startup is 5% faster on Linux and 30% faster on macOS
This commit is contained in:
parent
79d37ae979
commit
a8f8d5b4bd
78
Lib/site.py
78
Lib/site.py
|
@ -124,7 +124,7 @@ def removeduppaths():
|
||||||
# if they only differ in case); turn relative paths into absolute
|
# if they only differ in case); turn relative paths into absolute
|
||||||
# paths.
|
# paths.
|
||||||
dir, dircase = makepath(dir)
|
dir, dircase = makepath(dir)
|
||||||
if not dircase in known_paths:
|
if dircase not in known_paths:
|
||||||
L.append(dir)
|
L.append(dir)
|
||||||
known_paths.add(dircase)
|
known_paths.add(dircase)
|
||||||
sys.path[:] = L
|
sys.path[:] = L
|
||||||
|
@ -234,6 +234,46 @@ def check_enableusersite():
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE: sysconfig and it's dependencies are relatively large but site module
|
||||||
|
# needs very limited part of them.
|
||||||
|
# To speedup startup time, we have copy of them.
|
||||||
|
#
|
||||||
|
# See https://bugs.python.org/issue29585
|
||||||
|
|
||||||
|
# Copy of sysconfig._getuserbase()
|
||||||
|
def _getuserbase():
|
||||||
|
env_base = os.environ.get("PYTHONUSERBASE", None)
|
||||||
|
if env_base:
|
||||||
|
return env_base
|
||||||
|
|
||||||
|
def joinuser(*args):
|
||||||
|
return os.path.expanduser(os.path.join(*args))
|
||||||
|
|
||||||
|
if os.name == "nt":
|
||||||
|
base = os.environ.get("APPDATA") or "~"
|
||||||
|
return joinuser(base, "Python")
|
||||||
|
|
||||||
|
if sys.platform == "darwin" and sys._framework:
|
||||||
|
return joinuser("~", "Library", sys._framework,
|
||||||
|
"%d.%d" % sys.version_info[:2])
|
||||||
|
|
||||||
|
return joinuser("~", ".local")
|
||||||
|
|
||||||
|
|
||||||
|
# Same to sysconfig.get_path('purelib', os.name+'_user')
|
||||||
|
def _get_path(userbase):
|
||||||
|
version = sys.version_info
|
||||||
|
|
||||||
|
if os.name == 'nt':
|
||||||
|
return f'{userbase}/Python{version[0]}{version[1]}/site-packages'
|
||||||
|
|
||||||
|
if sys.platform == 'darwin' and sys._framework:
|
||||||
|
return f'{userbase}/lib/python/site-packages'
|
||||||
|
|
||||||
|
return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
|
||||||
|
|
||||||
|
|
||||||
def getuserbase():
|
def getuserbase():
|
||||||
"""Returns the `user base` directory path.
|
"""Returns the `user base` directory path.
|
||||||
|
|
||||||
|
@ -242,12 +282,11 @@ def getuserbase():
|
||||||
it.
|
it.
|
||||||
"""
|
"""
|
||||||
global USER_BASE
|
global USER_BASE
|
||||||
if USER_BASE is not None:
|
if USER_BASE is None:
|
||||||
return USER_BASE
|
USER_BASE = _getuserbase()
|
||||||
from sysconfig import get_config_var
|
|
||||||
USER_BASE = get_config_var('userbase')
|
|
||||||
return USER_BASE
|
return USER_BASE
|
||||||
|
|
||||||
|
|
||||||
def getusersitepackages():
|
def getusersitepackages():
|
||||||
"""Returns the user-specific site-packages directory path.
|
"""Returns the user-specific site-packages directory path.
|
||||||
|
|
||||||
|
@ -255,20 +294,11 @@ def getusersitepackages():
|
||||||
function will also set it.
|
function will also set it.
|
||||||
"""
|
"""
|
||||||
global USER_SITE
|
global USER_SITE
|
||||||
user_base = getuserbase() # this will also set USER_BASE
|
userbase = getuserbase() # this will also set USER_BASE
|
||||||
|
|
||||||
if USER_SITE is not None:
|
if USER_SITE is None:
|
||||||
return USER_SITE
|
USER_SITE = _get_path(userbase)
|
||||||
|
|
||||||
from sysconfig import get_path
|
|
||||||
|
|
||||||
if sys.platform == 'darwin':
|
|
||||||
from sysconfig import get_config_var
|
|
||||||
if get_config_var('PYTHONFRAMEWORK'):
|
|
||||||
USER_SITE = get_path('purelib', 'osx_framework_user')
|
|
||||||
return USER_SITE
|
|
||||||
|
|
||||||
USER_SITE = get_path('purelib', '%s_user' % os.name)
|
|
||||||
return USER_SITE
|
return USER_SITE
|
||||||
|
|
||||||
def addusersitepackages(known_paths):
|
def addusersitepackages(known_paths):
|
||||||
|
@ -310,15 +340,11 @@ def getsitepackages(prefixes=None):
|
||||||
else:
|
else:
|
||||||
sitepackages.append(prefix)
|
sitepackages.append(prefix)
|
||||||
sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
|
sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
|
||||||
if sys.platform == "darwin":
|
# for framework builds *only* we add the standard Apple locations.
|
||||||
# for framework builds *only* we add the standard Apple
|
if sys.platform == "darwin" and sys._framework:
|
||||||
# locations.
|
sitepackages.append(
|
||||||
from sysconfig import get_config_var
|
os.path.join("/Library", framework,
|
||||||
framework = get_config_var("PYTHONFRAMEWORK")
|
'%d.%d' % sys.version_info[:2], "site-packages"))
|
||||||
if framework:
|
|
||||||
sitepackages.append(
|
|
||||||
os.path.join("/Library", framework,
|
|
||||||
'%d.%d' % sys.version_info[:2], "site-packages"))
|
|
||||||
return sitepackages
|
return sitepackages
|
||||||
|
|
||||||
def addsitepackages(known_paths, prefixes=None):
|
def addsitepackages(known_paths, prefixes=None):
|
||||||
|
|
|
@ -51,6 +51,7 @@ _INSTALL_SCHEMES = {
|
||||||
'scripts': '{base}/Scripts',
|
'scripts': '{base}/Scripts',
|
||||||
'data': '{base}',
|
'data': '{base}',
|
||||||
},
|
},
|
||||||
|
# NOTE: When modifying "purelib" scheme, update site._get_path() too.
|
||||||
'nt_user': {
|
'nt_user': {
|
||||||
'stdlib': '{userbase}/Python{py_version_nodot}',
|
'stdlib': '{userbase}/Python{py_version_nodot}',
|
||||||
'platstdlib': '{userbase}/Python{py_version_nodot}',
|
'platstdlib': '{userbase}/Python{py_version_nodot}',
|
||||||
|
@ -177,32 +178,25 @@ def _get_default_scheme():
|
||||||
return os.name
|
return os.name
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE: site.py has copy of this function.
|
||||||
|
# Sync it when modify this function.
|
||||||
def _getuserbase():
|
def _getuserbase():
|
||||||
env_base = os.environ.get("PYTHONUSERBASE", None)
|
env_base = os.environ.get("PYTHONUSERBASE", None)
|
||||||
|
if env_base:
|
||||||
|
return env_base
|
||||||
|
|
||||||
def joinuser(*args):
|
def joinuser(*args):
|
||||||
return os.path.expanduser(os.path.join(*args))
|
return os.path.expanduser(os.path.join(*args))
|
||||||
|
|
||||||
if os.name == "nt":
|
if os.name == "nt":
|
||||||
base = os.environ.get("APPDATA") or "~"
|
base = os.environ.get("APPDATA") or "~"
|
||||||
if env_base:
|
return joinuser(base, "Python")
|
||||||
return env_base
|
|
||||||
else:
|
|
||||||
return joinuser(base, "Python")
|
|
||||||
|
|
||||||
if sys.platform == "darwin":
|
if sys.platform == "darwin" and sys._framework:
|
||||||
framework = get_config_var("PYTHONFRAMEWORK")
|
return joinuser("~", "Library", sys._framework,
|
||||||
if framework:
|
"%d.%d" % sys.version_info[:2])
|
||||||
if env_base:
|
|
||||||
return env_base
|
|
||||||
else:
|
|
||||||
return joinuser("~", "Library", framework, "%d.%d" %
|
|
||||||
sys.version_info[:2])
|
|
||||||
|
|
||||||
if env_base:
|
return joinuser("~", ".local")
|
||||||
return env_base
|
|
||||||
else:
|
|
||||||
return joinuser("~", ".local")
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_makefile(filename, vars=None):
|
def _parse_makefile(filename, vars=None):
|
||||||
|
|
|
@ -180,6 +180,13 @@ class HelperFunctionsTests(unittest.TestCase):
|
||||||
finally:
|
finally:
|
||||||
pth_file.cleanup()
|
pth_file.cleanup()
|
||||||
|
|
||||||
|
def test_getuserbase(self):
|
||||||
|
self.assertEqual(site._getuserbase(), sysconfig._getuserbase())
|
||||||
|
|
||||||
|
def test_get_path(self):
|
||||||
|
self.assertEqual(site._get_path(site._getuserbase()),
|
||||||
|
sysconfig.get_path('purelib', os.name + '_user'))
|
||||||
|
|
||||||
@unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
|
@unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
|
||||||
"user-site (site.ENABLE_USER_SITE)")
|
"user-site (site.ENABLE_USER_SITE)")
|
||||||
def test_s_option(self):
|
def test_s_option(self):
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
Avoid importing ``sysconfig`` from ``site`` to improve startup speed. Python
|
||||||
|
startup is about 5% faster on Linux and 30% faster on macOS.
|
|
@ -1965,6 +1965,7 @@ _PySys_BeginInit(void)
|
||||||
SET_SYS_FROM_STRING("_git",
|
SET_SYS_FROM_STRING("_git",
|
||||||
Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
|
Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
|
||||||
_Py_gitversion()));
|
_Py_gitversion()));
|
||||||
|
SET_SYS_FROM_STRING("_framework", PyUnicode_FromString(PYTHONFRAMEWORK));
|
||||||
SET_SYS_FROM_STRING("api_version",
|
SET_SYS_FROM_STRING("api_version",
|
||||||
PyLong_FromLong(PYTHON_API_VERSION));
|
PyLong_FromLong(PYTHON_API_VERSION));
|
||||||
SET_SYS_FROM_STRING("copyright",
|
SET_SYS_FROM_STRING("copyright",
|
||||||
|
|
|
@ -783,6 +783,7 @@ infodir
|
||||||
docdir
|
docdir
|
||||||
oldincludedir
|
oldincludedir
|
||||||
includedir
|
includedir
|
||||||
|
runstatedir
|
||||||
localstatedir
|
localstatedir
|
||||||
sharedstatedir
|
sharedstatedir
|
||||||
sysconfdir
|
sysconfdir
|
||||||
|
@ -896,6 +897,7 @@ datadir='${datarootdir}'
|
||||||
sysconfdir='${prefix}/etc'
|
sysconfdir='${prefix}/etc'
|
||||||
sharedstatedir='${prefix}/com'
|
sharedstatedir='${prefix}/com'
|
||||||
localstatedir='${prefix}/var'
|
localstatedir='${prefix}/var'
|
||||||
|
runstatedir='${localstatedir}/run'
|
||||||
includedir='${prefix}/include'
|
includedir='${prefix}/include'
|
||||||
oldincludedir='/usr/include'
|
oldincludedir='/usr/include'
|
||||||
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
|
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
|
||||||
|
@ -1148,6 +1150,15 @@ do
|
||||||
| -silent | --silent | --silen | --sile | --sil)
|
| -silent | --silent | --silen | --sile | --sil)
|
||||||
silent=yes ;;
|
silent=yes ;;
|
||||||
|
|
||||||
|
-runstatedir | --runstatedir | --runstatedi | --runstated \
|
||||||
|
| --runstate | --runstat | --runsta | --runst | --runs \
|
||||||
|
| --run | --ru | --r)
|
||||||
|
ac_prev=runstatedir ;;
|
||||||
|
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
|
||||||
|
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
|
||||||
|
| --run=* | --ru=* | --r=*)
|
||||||
|
runstatedir=$ac_optarg ;;
|
||||||
|
|
||||||
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
|
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
|
||||||
ac_prev=sbindir ;;
|
ac_prev=sbindir ;;
|
||||||
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
|
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
|
||||||
|
@ -1285,7 +1296,7 @@ fi
|
||||||
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
|
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
|
||||||
datadir sysconfdir sharedstatedir localstatedir includedir \
|
datadir sysconfdir sharedstatedir localstatedir includedir \
|
||||||
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
|
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
|
||||||
libdir localedir mandir
|
libdir localedir mandir runstatedir
|
||||||
do
|
do
|
||||||
eval ac_val=\$$ac_var
|
eval ac_val=\$$ac_var
|
||||||
# Remove trailing slashes.
|
# Remove trailing slashes.
|
||||||
|
@ -1438,6 +1449,7 @@ Fine tuning of the installation directories:
|
||||||
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
|
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
|
||||||
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
|
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
|
||||||
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
|
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
|
||||||
|
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
|
||||||
--libdir=DIR object code libraries [EPREFIX/lib]
|
--libdir=DIR object code libraries [EPREFIX/lib]
|
||||||
--includedir=DIR C header files [PREFIX/include]
|
--includedir=DIR C header files [PREFIX/include]
|
||||||
--oldincludedir=DIR C header files for non-gcc [/usr/include]
|
--oldincludedir=DIR C header files for non-gcc [/usr/include]
|
||||||
|
@ -3231,6 +3243,12 @@ fi
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
cat >>confdefs.h <<_ACEOF
|
||||||
|
#define PYTHONFRAMEWORK "${PYTHONFRAMEWORK}"
|
||||||
|
_ACEOF
|
||||||
|
|
||||||
|
|
||||||
##AC_ARG_WITH(dyld,
|
##AC_ARG_WITH(dyld,
|
||||||
## AS_HELP_STRING([--with-dyld],
|
## AS_HELP_STRING([--with-dyld],
|
||||||
## [Use (OpenStep|Rhapsody) dynamic linker]))
|
## [Use (OpenStep|Rhapsody) dynamic linker]))
|
||||||
|
|
|
@ -355,6 +355,8 @@ AC_SUBST(FRAMEWORKPYTHONW)
|
||||||
AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX)
|
AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX)
|
||||||
AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX)
|
AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX)
|
||||||
|
|
||||||
|
AC_DEFINE_UNQUOTED(PYTHONFRAMEWORK, "${PYTHONFRAMEWORK}", [framework name])
|
||||||
|
|
||||||
##AC_ARG_WITH(dyld,
|
##AC_ARG_WITH(dyld,
|
||||||
## AS_HELP_STRING([--with-dyld],
|
## AS_HELP_STRING([--with-dyld],
|
||||||
## [Use (OpenStep|Rhapsody) dynamic linker]))
|
## [Use (OpenStep|Rhapsody) dynamic linker]))
|
||||||
|
|
|
@ -1247,6 +1247,9 @@
|
||||||
/* Define as the preferred size in bits of long digits */
|
/* Define as the preferred size in bits of long digits */
|
||||||
#undef PYLONG_BITS_IN_DIGIT
|
#undef PYLONG_BITS_IN_DIGIT
|
||||||
|
|
||||||
|
/* framework name */
|
||||||
|
#undef PYTHONFRAMEWORK
|
||||||
|
|
||||||
/* Define if you want to coerce the C locale to a UTF-8 based locale */
|
/* Define if you want to coerce the C locale to a UTF-8 based locale */
|
||||||
#undef PY_COERCE_C_LOCALE
|
#undef PY_COERCE_C_LOCALE
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue