bpo-32030: Add pymain_get_global_config() (#4735)

* Py_Main() now starts by reading Py_xxx configuration variables to
  only work on its own private structure, and then later writes back
  the configuration into these variables.
* Replace Py_GETENV() with pymain_get_env_var() which ignores empty
  variables.
* Add _PyCoreConfig.dump_refs
* Add _PyCoreConfig.malloc_stats
* _PyObject_DebugMallocStats() is now responsible to check if debug
  hooks are installed. The function returns 1 if stats were written,
  or 0 if the hooks are disabled. Mark _PyMem_PymallocEnabled() as
  static.
This commit is contained in:
Victor Stinner 2017-12-06 17:26:10 +01:00 committed by GitHub
parent 672b6baa71
commit 6bf992a1ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 141 additions and 76 deletions

View File

@ -109,7 +109,7 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
/* Macros */ /* Macros */
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
#ifndef Py_LIMITED_API #ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out);
#endif /* #ifndef Py_LIMITED_API */ #endif /* #ifndef Py_LIMITED_API */
#endif #endif

View File

@ -24,10 +24,6 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);
/* Try to get the allocators name set by _PyMem_SetupAllocators(). */ /* Try to get the allocators name set by _PyMem_SetupAllocators(). */
PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void); PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);
#ifdef WITH_PYMALLOC
PyAPI_FUNC(int) _PyMem_PymallocEnabled(void);
#endif
/* Track an allocated memory block in the tracemalloc module. /* Track an allocated memory block in the tracemalloc module.
Return 0 on success, return -1 on error (failed to allocate memory to store Return 0 on success, return -1 on error (failed to allocate memory to store
the trace). the trace).

View File

@ -36,6 +36,8 @@ typedef struct {
int import_time; /* -X importtime */ int import_time; /* -X importtime */
int show_ref_count; /* -X showrefcount */ int show_ref_count; /* -X showrefcount */
int show_alloc_count; /* -X showalloccount */ int show_alloc_count; /* -X showalloccount */
int dump_refs; /* PYTHONDUMPREFS */
int malloc_stats; /* PYTHONMALLOCSTATS */
} _PyCoreConfig; } _PyCoreConfig;
#define _PyCoreConfig_INIT (_PyCoreConfig){.use_hash_seed = -1} #define _PyCoreConfig_INIT (_PyCoreConfig){.use_hash_seed = -1}
@ -111,7 +113,7 @@ typedef struct _is {
PyObject *after_forkers_child; PyObject *after_forkers_child;
#endif #endif
} PyInterpreterState; } PyInterpreterState;
#endif #endif /* !Py_LIMITED_API */
/* State unique per thread */ /* State unique per thread */
@ -133,7 +135,7 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
#define PyTrace_C_EXCEPTION 5 #define PyTrace_C_EXCEPTION 5
#define PyTrace_C_RETURN 6 #define PyTrace_C_RETURN 6
#define PyTrace_OPCODE 7 #define PyTrace_OPCODE 7
#endif #endif /* Py_LIMITED_API */
#ifdef Py_LIMITED_API #ifdef Py_LIMITED_API
typedef struct _ts PyThreadState; typedef struct _ts PyThreadState;
@ -238,7 +240,7 @@ typedef struct _ts {
/* XXX signal handlers should also be here */ /* XXX signal handlers should also be here */
} PyThreadState; } PyThreadState;
#endif #endif /* !Py_LIMITED_API */
PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
@ -363,7 +365,7 @@ PyAPI_FUNC(int) PyGILState_Check(void);
Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini() Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini()
is called. */ is called. */
PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void);
#endif #endif /* !Py_LIMITED_API */
/* The implementation of sys._current_frames() Returns a dict mapping /* The implementation of sys._current_frames() Returns a dict mapping

View File

@ -162,8 +162,8 @@ pymain_get_env_var(const char *name)
static void static void
pymain_run_startup(PyCompilerFlags *cf) pymain_run_startup(PyCompilerFlags *cf)
{ {
char *startup = Py_GETENV("PYTHONSTARTUP"); char *startup = pymain_get_env_var("PYTHONSTARTUP");
if (startup == NULL || startup[0] == '\0') { if (startup == NULL) {
return; return;
} }
@ -377,23 +377,28 @@ typedef struct {
wchar_t *command; /* -c argument */ wchar_t *command; /* -c argument */
wchar_t *module; /* -m argument */ wchar_t *module; /* -m argument */
_Py_OptList warning_options; /* -W options */ _Py_OptList warning_options; /* -W options */
PyObject *extra_options; /* -X options */
int print_help; /* -h, -? options */ int print_help; /* -h, -? options */
int print_version; /* -V option */ int print_version; /* -V option */
int bytes_warning; /* Py_BytesWarningFlag */ int bytes_warning; /* Py_BytesWarningFlag, -b */
int debug; /* Py_DebugFlag */ int debug; /* Py_DebugFlag, -b, PYTHONDEBUG */
int inspect; /* Py_InspectFlag */ int inspect; /* Py_InspectFlag, -i, PYTHONINSPECT */
int interactive; /* Py_InteractiveFlag */ int interactive; /* Py_InteractiveFlag, -i */
int isolated; /* Py_IsolatedFlag */ int isolated; /* Py_IsolatedFlag, -I */
int optimization_level; /* Py_OptimizeFlag */ int optimization_level; /* Py_OptimizeFlag, -O, PYTHONOPTIMIZE */
int dont_write_bytecode; /* Py_DontWriteBytecodeFlag */ int dont_write_bytecode; /* Py_DontWriteBytecodeFlag, -B, PYTHONDONTWRITEBYTECODE */
int no_user_site_directory; /* Py_NoUserSiteDirectory */ int no_user_site_directory; /* Py_NoUserSiteDirectory, -I, -s, PYTHONNOUSERSITE */
int no_site_import; /* Py_NoSiteFlag */ int no_site_import; /* Py_NoSiteFlag, -S */
int use_unbuffered_io; /* Py_UnbufferedStdioFlag */ int use_unbuffered_io; /* Py_UnbufferedStdioFlag, -u, PYTHONUNBUFFERED */
int verbosity; /* Py_VerboseFlag */ int verbosity; /* Py_VerboseFlag, -v, PYTHONVERBOSE */
int quiet_flag; /* Py_QuietFlag */ int quiet_flag; /* Py_QuietFlag, -q */
int skip_first_line; /* -x option */ int skip_first_line; /* -x option */
_Py_OptList xoptions; /* -X options */ _Py_OptList xoptions; /* -X options */
#ifdef MS_WINDOWS
int legacy_windows_fs_encoding; /* Py_LegacyWindowsFSEncodingFlag,
PYTHONLEGACYWINDOWSFSENCODING */
int legacy_windows_stdio; /* Py_LegacyWindowsStdioFlag,
PYTHONLEGACYWINDOWSSTDIO */
#endif
} _Py_CommandLineDetails; } _Py_CommandLineDetails;
/* Structure used by Py_Main() to pass data to subfunctions */ /* Structure used by Py_Main() to pass data to subfunctions */
@ -695,19 +700,6 @@ pymain_parse_cmdline_impl(_PyMain *pymain)
} }
static void
maybe_set_flag(int *flag, int value)
{
/* Helper to set flag variables from command line options
* - uses the higher of the two values if they're both set
* - otherwise leaves the flag unset
*/
if (*flag < value) {
*flag = value;
}
}
static int static int
pymain_add_xoptions(_PyMain *pymain) pymain_add_xoptions(_PyMain *pymain)
{ {
@ -790,9 +782,8 @@ pymain_warnings_envvar(_PyMain *pymain)
PyMem_RawFree(buf); PyMem_RawFree(buf);
} }
#else #else
char *p; char *p = pymain_get_env_var("PYTHONWARNINGS");
if (p != NULL) {
if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
char *buf, *oldloc; char *buf, *oldloc;
/* settle for strtok here as there's no one standard /* settle for strtok here as there's no one standard
@ -885,7 +876,6 @@ config_get_program_name(_PyMainInterpreterConfig *config)
} }
#ifdef __APPLE__ #ifdef __APPLE__
char *p;
/* On MacOS X, when the Python interpreter is embedded in an /* On MacOS X, when the Python interpreter is embedded in an
application bundle, it gets executed by a bootstrapping script application bundle, it gets executed by a bootstrapping script
that does os.execve() with an argv[0] that's different from the that does os.execve() with an argv[0] that's different from the
@ -895,7 +885,8 @@ config_get_program_name(_PyMainInterpreterConfig *config)
so the actual executable path is passed in an environment variable. so the actual executable path is passed in an environment variable.
See Lib/plat-mac/bundlebuiler.py for details about the bootstrap See Lib/plat-mac/bundlebuiler.py for details about the bootstrap
script. */ script. */
if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') { char *p = pymain_get_env_var("PYTHONEXECUTABLE");
if (p != NULL) {
size_t len; size_t len;
wchar_t* program_name = Py_DecodeLocale(p, &len); wchar_t* program_name = Py_DecodeLocale(p, &len);
if (program_name == NULL) { if (program_name == NULL) {
@ -1015,25 +1006,76 @@ pymain_set_argv(_PyMain *pymain)
} }
static void
pymain_get_flag(int flag, int *value)
{
if (flag) {
*value = flag;
}
}
static void
pymain_set_flag(int *flag, int value)
{
/* Helper to set flag variables from command line options
* - uses the higher of the two values if they're both set
* - otherwise leaves the flag unset
*/
if (*flag < value) {
*flag = value;
}
}
/* Get Py_xxx global configuration variables */
static void
pymain_get_global_config(_PyMain *pymain)
{
_Py_CommandLineDetails *cmdline = &pymain->cmdline;
pymain_get_flag(Py_BytesWarningFlag, &cmdline->bytes_warning);
pymain_get_flag(Py_DebugFlag, &cmdline->debug);
pymain_get_flag(Py_InspectFlag, &cmdline->inspect);
pymain_get_flag(Py_InteractiveFlag, &cmdline->interactive);
pymain_get_flag(Py_IsolatedFlag, &cmdline->isolated);
pymain_get_flag(Py_OptimizeFlag, &cmdline->optimization_level);
pymain_get_flag(Py_DontWriteBytecodeFlag, &cmdline->dont_write_bytecode);
pymain_get_flag(Py_NoUserSiteDirectory, &cmdline->no_user_site_directory);
pymain_get_flag(Py_NoSiteFlag, &cmdline->no_site_import);
pymain_get_flag(Py_UnbufferedStdioFlag, &cmdline->use_unbuffered_io);
pymain_get_flag(Py_VerboseFlag, &cmdline->verbosity);
pymain_get_flag(Py_QuietFlag, &cmdline->quiet_flag);
#ifdef MS_WINDOWS
pymain_get_flag(Py_LegacyWindowsFSEncodingFlag, &cmdline->legacy_windows_fs_encoding);
pymain_get_flag(Py_LegacyWindowsStdioFlag, &cmdline->legacy_windows_stdio);
#endif
pymain_get_flag(Py_IgnoreEnvironmentFlag, &pymain->core_config.ignore_environment);
}
/* Set Py_XXX global configuration variables */ /* Set Py_XXX global configuration variables */
static void static void
pymain_set_global_config(_PyMain *pymain) pymain_set_global_config(_PyMain *pymain)
{ {
_Py_CommandLineDetails *cmdline = &pymain->cmdline; _Py_CommandLineDetails *cmdline = &pymain->cmdline;
maybe_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning); pymain_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning);
maybe_set_flag(&Py_DebugFlag, cmdline->debug); pymain_set_flag(&Py_DebugFlag, cmdline->debug);
maybe_set_flag(&Py_InspectFlag, cmdline->inspect); pymain_set_flag(&Py_InspectFlag, cmdline->inspect);
maybe_set_flag(&Py_InteractiveFlag, cmdline->interactive); pymain_set_flag(&Py_InteractiveFlag, cmdline->interactive);
maybe_set_flag(&Py_IsolatedFlag, cmdline->isolated); pymain_set_flag(&Py_IsolatedFlag, cmdline->isolated);
maybe_set_flag(&Py_OptimizeFlag, cmdline->optimization_level); pymain_set_flag(&Py_OptimizeFlag, cmdline->optimization_level);
maybe_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode); pymain_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode);
maybe_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory); pymain_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory);
maybe_set_flag(&Py_NoSiteFlag, cmdline->no_site_import); pymain_set_flag(&Py_NoSiteFlag, cmdline->no_site_import);
maybe_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io); pymain_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io);
maybe_set_flag(&Py_VerboseFlag, cmdline->verbosity); pymain_set_flag(&Py_VerboseFlag, cmdline->verbosity);
maybe_set_flag(&Py_QuietFlag, cmdline->quiet_flag); pymain_set_flag(&Py_QuietFlag, cmdline->quiet_flag);
#ifdef MS_WINDOWS
pymain_set_flag(&Py_LegacyWindowsFSEncodingFlag, cmdline->legacy_windows_fs_encoding);
pymain_set_flag(&Py_LegacyWindowsStdioFlag, cmdline->legacy_windows_stdio);
#endif
maybe_set_flag(&Py_IgnoreEnvironmentFlag, pymain->core_config.ignore_environment); pymain_set_flag(&Py_IgnoreEnvironmentFlag, pymain->core_config.ignore_environment);
} }
@ -1330,24 +1372,25 @@ pymain_set_flag_from_env(int *flag, const char *name)
static void static void
pymain_set_flags_from_env(_PyMain *pymain) pymain_set_flags_from_env(_PyMain *pymain)
{ {
pymain_set_flag_from_env(&Py_DebugFlag, _Py_CommandLineDetails *cmdline = &pymain->cmdline;
pymain_set_flag_from_env(&cmdline->debug,
"PYTHONDEBUG"); "PYTHONDEBUG");
pymain_set_flag_from_env(&Py_VerboseFlag, pymain_set_flag_from_env(&cmdline->verbosity,
"PYTHONVERBOSE"); "PYTHONVERBOSE");
pymain_set_flag_from_env(&Py_OptimizeFlag, pymain_set_flag_from_env(&cmdline->optimization_level,
"PYTHONOPTIMIZE"); "PYTHONOPTIMIZE");
pymain_set_flag_from_env(&Py_InspectFlag, pymain_set_flag_from_env(&cmdline->inspect,
"PYTHONINSPECT"); "PYTHONINSPECT");
pymain_set_flag_from_env(&Py_DontWriteBytecodeFlag, pymain_set_flag_from_env(&cmdline->dont_write_bytecode,
"PYTHONDONTWRITEBYTECODE"); "PYTHONDONTWRITEBYTECODE");
pymain_set_flag_from_env(&Py_NoUserSiteDirectory, pymain_set_flag_from_env(&cmdline->no_user_site_directory,
"PYTHONNOUSERSITE"); "PYTHONNOUSERSITE");
pymain_set_flag_from_env(&Py_UnbufferedStdioFlag, pymain_set_flag_from_env(&cmdline->use_unbuffered_io,
"PYTHONUNBUFFERED"); "PYTHONUNBUFFERED");
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
pymain_set_flag_from_env(&Py_LegacyWindowsFSEncodingFlag, pymain_set_flag_from_env(&cmdline->legacy_windows_fs_encoding,
"PYTHONLEGACYWINDOWSFSENCODING"); "PYTHONLEGACYWINDOWSFSENCODING");
pymain_set_flag_from_env(&Py_LegacyWindowsStdioFlag, pymain_set_flag_from_env(&cmdline->legacy_windows_stdio,
"PYTHONLEGACYWINDOWSSTDIO"); "PYTHONLEGACYWINDOWSSTDIO");
#endif #endif
} }
@ -1485,7 +1528,7 @@ pymain_parse_envvars(_PyMain *pymain)
return -1; return -1;
} }
core_config->allocator = Py_GETENV("PYTHONMALLOC"); core_config->allocator = pymain_get_env_var("PYTHONMALLOC");
/* -X options */ /* -X options */
if (pymain_get_xoption(pymain, L"showrefcount")) { if (pymain_get_xoption(pymain, L"showrefcount")) {
@ -1514,6 +1557,14 @@ pymain_parse_envvars(_PyMain *pymain)
core_config->faulthandler = 1; core_config->faulthandler = 1;
core_config->allocator = "debug"; core_config->allocator = "debug";
} }
if (pymain_get_env_var("PYTHONDUMPREFS")) {
pymain->core_config.dump_refs = 1;
}
if (pymain_get_env_var("PYTHONMALLOCSTATS")) {
pymain->core_config.malloc_stats = 1;
}
return 0; return 0;
} }
@ -1535,6 +1586,7 @@ pymain_parse_cmdline_envvars_impl(_PyMain *pymain)
return 1; return 1;
} }
/* Set Py_IgnoreEnvironmentFlag needed by Py_GETENV() */
pymain_set_global_config(pymain); pymain_set_global_config(pymain);
if (pymain_parse_envvars(pymain) < 0) { if (pymain_parse_envvars(pymain) < 0) {
@ -1568,6 +1620,8 @@ pymain_parse_cmdline_envvars(_PyMain *pymain)
static int static int
pymain_init_python(_PyMain *pymain) pymain_init_python(_PyMain *pymain)
{ {
pymain_set_global_config(pymain);
pymain_init_stdio(pymain); pymain_init_stdio(pymain);
pymain->err = _Py_InitializeCore(&pymain->core_config); pymain->err = _Py_InitializeCore(&pymain->core_config);
@ -1641,6 +1695,8 @@ pymain_impl(_PyMain *pymain)
return -1; return -1;
} }
pymain_get_global_config(pymain);
res = pymain_parse_cmdline_envvars(pymain); res = pymain_parse_cmdline_envvars(pymain);
if (res < 0) { if (res < 0) {
return -1; return -1;

View File

@ -379,7 +379,7 @@ _PyMem_DebugEnabled(void)
return (_PyObject.malloc == _PyMem_DebugMalloc); return (_PyObject.malloc == _PyMem_DebugMalloc);
} }
int static int
_PyMem_PymallocEnabled(void) _PyMem_PymallocEnabled(void)
{ {
if (_PyMem_DebugEnabled()) { if (_PyMem_DebugEnabled()) {
@ -2467,10 +2467,17 @@ pool_is_in_list(const poolp target, poolp list)
/* Print summary info to "out" about the state of pymalloc's structures. /* Print summary info to "out" about the state of pymalloc's structures.
* In Py_DEBUG mode, also perform some expensive internal consistency * In Py_DEBUG mode, also perform some expensive internal consistency
* checks. * checks.
*
* Return 0 if the memory debug hooks are not installed or no statistics was
* writen into out, return 1 otherwise.
*/ */
void int
_PyObject_DebugMallocStats(FILE *out) _PyObject_DebugMallocStats(FILE *out)
{ {
if (!_PyMem_PymallocEnabled()) {
return 0;
}
uint i; uint i;
const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT;
/* # of pools, allocated blocks, and free blocks per class index */ /* # of pools, allocated blocks, and free blocks per class index */
@ -2603,6 +2610,7 @@ _PyObject_DebugMallocStats(FILE *out)
total += printone(out, "# bytes lost to quantization", quantization); total += printone(out, "# bytes lost to quantization", quantization);
total += printone(out, "# bytes lost to arena alignment", arena_alignment); total += printone(out, "# bytes lost to arena alignment", arena_alignment);
(void)printone(out, "Total", total); (void)printone(out, "Total", total);
return 1;
} }
#endif /* #ifdef WITH_PYMALLOC */ #endif /* #ifdef WITH_PYMALLOC */

View File

@ -1103,6 +1103,10 @@ Py_FinalizeEx(void)
tstate = PyThreadState_GET(); tstate = PyThreadState_GET();
interp = tstate->interp; interp = tstate->interp;
/* Copy the core config to be able to use it even
after PyInterpreterState_Delete() */
_PyCoreConfig core_config = interp->core_config;
/* Remaining threads (e.g. daemon threads) will automatically exit /* Remaining threads (e.g. daemon threads) will automatically exit
after taking the GIL (in PyEval_RestoreThread()). */ after taking the GIL (in PyEval_RestoreThread()). */
_PyRuntime.finalizing = tstate; _PyRuntime.finalizing = tstate;
@ -1186,7 +1190,7 @@ Py_FinalizeEx(void)
_PyHash_Fini(); _PyHash_Fini();
#ifdef Py_REF_DEBUG #ifdef Py_REF_DEBUG
if (interp->core_config.show_ref_count) { if (core_config.show_ref_count) {
_PyDebug_PrintTotalRefs(); _PyDebug_PrintTotalRefs();
} }
#endif #endif
@ -1197,8 +1201,9 @@ Py_FinalizeEx(void)
* Alas, a lot of stuff may still be alive now that will be cleaned * Alas, a lot of stuff may still be alive now that will be cleaned
* up later. * up later.
*/ */
if (Py_GETENV("PYTHONDUMPREFS")) if (core_config.dump_refs) {
_Py_PrintReferences(stderr); _Py_PrintReferences(stderr);
}
#endif /* Py_TRACE_REFS */ #endif /* Py_TRACE_REFS */
/* Clear interpreter state and all thread states. */ /* Clear interpreter state and all thread states. */
@ -1260,14 +1265,13 @@ Py_FinalizeEx(void)
* An address can be used to find the repr of the object, printed * An address can be used to find the repr of the object, printed
* above by _Py_PrintReferences. * above by _Py_PrintReferences.
*/ */
if (Py_GETENV("PYTHONDUMPREFS")) if (core_config.dump_refs) {
_Py_PrintReferenceAddresses(stderr); _Py_PrintReferenceAddresses(stderr);
}
#endif /* Py_TRACE_REFS */ #endif /* Py_TRACE_REFS */
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
if (_PyMem_PymallocEnabled()) { if (core_config.malloc_stats) {
char *opt = Py_GETENV("PYTHONMALLOCSTATS"); _PyObject_DebugMallocStats(stderr);
if (opt != NULL && *opt != '\0')
_PyObject_DebugMallocStats(stderr);
} }
#endif #endif

View File

@ -1368,8 +1368,7 @@ static PyObject *
sys_debugmallocstats(PyObject *self, PyObject *args) sys_debugmallocstats(PyObject *self, PyObject *args)
{ {
#ifdef WITH_PYMALLOC #ifdef WITH_PYMALLOC
if (_PyMem_PymallocEnabled()) { if (_PyObject_DebugMallocStats(stderr)) {
_PyObject_DebugMallocStats(stderr);
fputc('\n', stderr); fputc('\n', stderr);
} }
#endif #endif