bpo-29585: optimize site.py startup time (GH-136)

Avoid importing `sysconfig` from `site` by copying minimum code.
Python startup is 5% faster on Linux and 30% faster on macOS
This commit is contained in:
INADA Naoki 2017-06-29 00:31:53 +09:00 committed by GitHub
parent 79d37ae979
commit a8f8d5b4bd
8 changed files with 96 additions and 43 deletions

View File

@ -124,7 +124,7 @@ def removeduppaths():
# if they only differ in case); turn relative paths into absolute # if they only differ in case); turn relative paths into absolute
# paths. # paths.
dir, dircase = makepath(dir) dir, dircase = makepath(dir)
if not dircase in known_paths: if dircase not in known_paths:
L.append(dir) L.append(dir)
known_paths.add(dircase) known_paths.add(dircase)
sys.path[:] = L sys.path[:] = L
@ -234,6 +234,46 @@ def check_enableusersite():
return True return True
# NOTE: sysconfig and it's dependencies are relatively large but site module
# needs very limited part of them.
# To speedup startup time, we have copy of them.
#
# See https://bugs.python.org/issue29585
# Copy of sysconfig._getuserbase()
def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None)
if env_base:
return env_base
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
return joinuser(base, "Python")
if sys.platform == "darwin" and sys._framework:
return joinuser("~", "Library", sys._framework,
"%d.%d" % sys.version_info[:2])
return joinuser("~", ".local")
# Same to sysconfig.get_path('purelib', os.name+'_user')
def _get_path(userbase):
version = sys.version_info
if os.name == 'nt':
return f'{userbase}/Python{version[0]}{version[1]}/site-packages'
if sys.platform == 'darwin' and sys._framework:
return f'{userbase}/lib/python/site-packages'
return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
def getuserbase(): def getuserbase():
"""Returns the `user base` directory path. """Returns the `user base` directory path.
@ -242,12 +282,11 @@ def getuserbase():
it. it.
""" """
global USER_BASE global USER_BASE
if USER_BASE is not None: if USER_BASE is None:
return USER_BASE USER_BASE = _getuserbase()
from sysconfig import get_config_var
USER_BASE = get_config_var('userbase')
return USER_BASE return USER_BASE
def getusersitepackages(): def getusersitepackages():
"""Returns the user-specific site-packages directory path. """Returns the user-specific site-packages directory path.
@ -255,20 +294,11 @@ def getusersitepackages():
function will also set it. function will also set it.
""" """
global USER_SITE global USER_SITE
user_base = getuserbase() # this will also set USER_BASE userbase = getuserbase() # this will also set USER_BASE
if USER_SITE is not None: if USER_SITE is None:
return USER_SITE USER_SITE = _get_path(userbase)
from sysconfig import get_path
if sys.platform == 'darwin':
from sysconfig import get_config_var
if get_config_var('PYTHONFRAMEWORK'):
USER_SITE = get_path('purelib', 'osx_framework_user')
return USER_SITE
USER_SITE = get_path('purelib', '%s_user' % os.name)
return USER_SITE return USER_SITE
def addusersitepackages(known_paths): def addusersitepackages(known_paths):
@ -310,15 +340,11 @@ def getsitepackages(prefixes=None):
else: else:
sitepackages.append(prefix) sitepackages.append(prefix)
sitepackages.append(os.path.join(prefix, "lib", "site-packages")) sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
if sys.platform == "darwin": # for framework builds *only* we add the standard Apple locations.
# for framework builds *only* we add the standard Apple if sys.platform == "darwin" and sys._framework:
# locations. sitepackages.append(
from sysconfig import get_config_var os.path.join("/Library", framework,
framework = get_config_var("PYTHONFRAMEWORK") '%d.%d' % sys.version_info[:2], "site-packages"))
if framework:
sitepackages.append(
os.path.join("/Library", framework,
'%d.%d' % sys.version_info[:2], "site-packages"))
return sitepackages return sitepackages
def addsitepackages(known_paths, prefixes=None): def addsitepackages(known_paths, prefixes=None):

View File

@ -51,6 +51,7 @@ _INSTALL_SCHEMES = {
'scripts': '{base}/Scripts', 'scripts': '{base}/Scripts',
'data': '{base}', 'data': '{base}',
}, },
# NOTE: When modifying "purelib" scheme, update site._get_path() too.
'nt_user': { 'nt_user': {
'stdlib': '{userbase}/Python{py_version_nodot}', 'stdlib': '{userbase}/Python{py_version_nodot}',
'platstdlib': '{userbase}/Python{py_version_nodot}', 'platstdlib': '{userbase}/Python{py_version_nodot}',
@ -177,32 +178,25 @@ def _get_default_scheme():
return os.name return os.name
# NOTE: site.py has copy of this function.
# Sync it when modify this function.
def _getuserbase(): def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None) env_base = os.environ.get("PYTHONUSERBASE", None)
if env_base:
return env_base
def joinuser(*args): def joinuser(*args):
return os.path.expanduser(os.path.join(*args)) return os.path.expanduser(os.path.join(*args))
if os.name == "nt": if os.name == "nt":
base = os.environ.get("APPDATA") or "~" base = os.environ.get("APPDATA") or "~"
if env_base: return joinuser(base, "Python")
return env_base
else:
return joinuser(base, "Python")
if sys.platform == "darwin": if sys.platform == "darwin" and sys._framework:
framework = get_config_var("PYTHONFRAMEWORK") return joinuser("~", "Library", sys._framework,
if framework: "%d.%d" % sys.version_info[:2])
if env_base:
return env_base
else:
return joinuser("~", "Library", framework, "%d.%d" %
sys.version_info[:2])
if env_base: return joinuser("~", ".local")
return env_base
else:
return joinuser("~", ".local")
def _parse_makefile(filename, vars=None): def _parse_makefile(filename, vars=None):

View File

@ -180,6 +180,13 @@ class HelperFunctionsTests(unittest.TestCase):
finally: finally:
pth_file.cleanup() pth_file.cleanup()
def test_getuserbase(self):
self.assertEqual(site._getuserbase(), sysconfig._getuserbase())
def test_get_path(self):
self.assertEqual(site._get_path(site._getuserbase()),
sysconfig.get_path('purelib', os.name + '_user'))
@unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 " @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
"user-site (site.ENABLE_USER_SITE)") "user-site (site.ENABLE_USER_SITE)")
def test_s_option(self): def test_s_option(self):

View File

@ -0,0 +1,2 @@
Avoid importing ``sysconfig`` from ``site`` to improve startup speed. Python
startup is about 5% faster on Linux and 30% faster on macOS.

View File

@ -1965,6 +1965,7 @@ _PySys_BeginInit(void)
SET_SYS_FROM_STRING("_git", SET_SYS_FROM_STRING("_git",
Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
_Py_gitversion())); _Py_gitversion()));
SET_SYS_FROM_STRING("_framework", PyUnicode_FromString(PYTHONFRAMEWORK));
SET_SYS_FROM_STRING("api_version", SET_SYS_FROM_STRING("api_version",
PyLong_FromLong(PYTHON_API_VERSION)); PyLong_FromLong(PYTHON_API_VERSION));
SET_SYS_FROM_STRING("copyright", SET_SYS_FROM_STRING("copyright",

20
configure vendored
View File

@ -783,6 +783,7 @@ infodir
docdir docdir
oldincludedir oldincludedir
includedir includedir
runstatedir
localstatedir localstatedir
sharedstatedir sharedstatedir
sysconfdir sysconfdir
@ -896,6 +897,7 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc' sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com' sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var' localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include' includedir='${prefix}/include'
oldincludedir='/usr/include' oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@ -1148,6 +1150,15 @@ do
| -silent | --silent | --silen | --sile | --sil) | -silent | --silent | --silen | --sile | --sil)
silent=yes ;; silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;; ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@ -1285,7 +1296,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \ datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir libdir localedir mandir runstatedir
do do
eval ac_val=\$$ac_var eval ac_val=\$$ac_var
# Remove trailing slashes. # Remove trailing slashes.
@ -1438,6 +1449,7 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var] --localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib] --libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include] --includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include] --oldincludedir=DIR C header files for non-gcc [/usr/include]
@ -3231,6 +3243,12 @@ fi
cat >>confdefs.h <<_ACEOF
#define PYTHONFRAMEWORK "${PYTHONFRAMEWORK}"
_ACEOF
##AC_ARG_WITH(dyld, ##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld], ## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker])) ## [Use (OpenStep|Rhapsody) dynamic linker]))

View File

@ -355,6 +355,8 @@ AC_SUBST(FRAMEWORKPYTHONW)
AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX) AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX)
AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX) AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX)
AC_DEFINE_UNQUOTED(PYTHONFRAMEWORK, "${PYTHONFRAMEWORK}", [framework name])
##AC_ARG_WITH(dyld, ##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld], ## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker])) ## [Use (OpenStep|Rhapsody) dynamic linker]))

View File

@ -1247,6 +1247,9 @@
/* Define as the preferred size in bits of long digits */ /* Define as the preferred size in bits of long digits */
#undef PYLONG_BITS_IN_DIGIT #undef PYLONG_BITS_IN_DIGIT
/* framework name */
#undef PYTHONFRAMEWORK
/* Define if you want to coerce the C locale to a UTF-8 based locale */ /* Define if you want to coerce the C locale to a UTF-8 based locale */
#undef PY_COERCE_C_LOCALE #undef PY_COERCE_C_LOCALE