From 93c92f7d1dbb6e7e472f1d0444c6968858113de2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Sep 2017 23:46:04 -0700 Subject: [PATCH] bpo-31404: Revert "remove modules from Py_InterpreterState (#1638)" (#3565) PR #1638, for bpo-28411, causes problems in some (very) edge cases. Until that gets sorted out, we're reverting the merge. PR #3506, a fix on top of #1638, is also getting reverted. --- Doc/c-api/import.rst | 7 - Doc/whatsnew/3.7.rst | 3 - Include/import.h | 25 +-- Include/modsupport.h | 4 - Include/object.h | 3 +- Include/pystate.h | 3 + .../2017-09-04-10-46-09.bpo-28411.IU9rQL.rst | 4 - Modules/_pickle.c | 12 +- Modules/pyexpat.c | 10 +- Objects/moduleobject.c | 12 +- Objects/object.c | 27 +-- Objects/typeobject.c | 3 +- Python/_warnings.c | 5 +- Python/bltinmodule.c | 2 +- Python/ceval.c | 2 +- Python/import.c | 205 ++++-------------- Python/importdl.c | 3 +- Python/pylifecycle.c | 76 +++---- Python/pystate.c | 4 + Python/pythonrun.c | 5 +- Python/sysmodule.c | 47 ++-- 21 files changed, 154 insertions(+), 308 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8cdc256e7c9..7c16ece0586 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -204,13 +204,6 @@ Importing Modules Return the dictionary used for the module administration (a.k.a. ``sys.modules``). Note that this is a per-interpreter variable. -.. c:function:: PyObject* PyImport_GetModule(PyObject *name) - - Return the already imported module with the given name. If the - module has not been imported yet then returns NULL but does not set - an error. Returns NULL and sets an error if the lookup failed. - - .. versionadded:: 3.7 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 6ff1bfcb68f..44911ccc354 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -486,9 +486,6 @@ Changes in the Python API and module are affected by this change. (Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.) -* ``PyInterpreterState`` no longer has a ``modules`` field. Instead use - ``sys.modules``. - * The *mode* argument of :func:`os.makedirs` no longer affects the file permission bits of newly-created intermediate-level directories. To set their file permission bits you can set the umask before invoking diff --git a/Include/import.h b/Include/import.h index 5463c0cb960..7e83985b3a4 100644 --- a/Include/import.h +++ b/Include/import.h @@ -38,25 +38,11 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( ); #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); -#endif -PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name); -PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name); -PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name); -PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); -#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyObject *name ); #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *); -#endif PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); @@ -106,19 +92,14 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void); PyAPI_FUNC(void) _PyImport_ReInitLock(void); PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin( - const char *name, /* UTF-8 encoded string */ - PyObject *modules + const char *name /* UTF-8 encoded string */ ); PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *, - PyObject *); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, - const char *name, /* UTF-8 encoded string */ - PyObject *modules + const char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); +PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); struct _inittab { const char *name; /* ASCII encoded string */ diff --git a/Include/modsupport.h b/Include/modsupport.h index 73d86a94b95..8c7cf39d9a3 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -191,10 +191,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, int apiver); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*, - int apiver); -#endif #ifdef Py_LIMITED_API #define PyModule_Create(module) \ diff --git a/Include/object.h b/Include/object.h index 9bb780e28bc..b46d4c30e1e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -727,13 +727,14 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); /* Py_REF_DEBUG also controls the display of refcounts and memory block * allocations at the interactive prompt and at interpreter shutdown */ -PyAPI_FUNC(PyObject *) _PyDebug_XOptionShowRefCount(void); PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void); +#define _PY_DEBUG_PRINT_TOTAL_REFS() _PyDebug_PrintTotalRefs() #else #define _Py_INC_REFTOTAL #define _Py_DEC_REFTOTAL #define _Py_REF_DEBUG_COMMA #define _Py_CHECK_REFCNT(OP) /* a semicolon */; +#define _PY_DEBUG_PRINT_TOTAL_REFS() #endif /* Py_REF_DEBUG */ #ifdef COUNT_ALLOCS diff --git a/Include/pystate.h b/Include/pystate.h index 5b75bb080fe..09061144605 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -53,6 +53,7 @@ typedef struct _is { int64_t id; + PyObject *modules; PyObject *modules_by_index; PyObject *sysdict; PyObject *builtins; @@ -60,6 +61,8 @@ typedef struct _is { /* Used in Python/sysmodule.c. */ int check_interval; + PyObject *warnoptions; + PyObject *xoptions; /* Used in Modules/_threadmodule.c. */ long num_threads; diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst deleted file mode 100644 index 2417f781276..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst +++ /dev/null @@ -1,4 +0,0 @@ -``PyInterpreterState`` has a "modules" field that is copied into -``sys.modules`` during interpreter startup. This causes problems if a -program replaces ``sys.modules`` with something else. To solve this we -eliminate ``PyInterpreterState.modules``. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index bcbe4ac7945..3165b4e6d68 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6425,7 +6425,9 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, /*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/ { PyObject *global; + PyObject *modules_dict; PyObject *module; + _Py_IDENTIFIER(modules); /* Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do this only with old pickle protocols and when the @@ -6482,7 +6484,13 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, } } - module = PyImport_GetModule(module_name); + modules_dict = _PySys_GetObjectId(&PyId_modules); + if (modules_dict == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); + return NULL; + } + + module = PyDict_GetItemWithError(modules_dict, module_name); if (module == NULL) { if (PyErr_Occurred()) return NULL; @@ -6490,11 +6498,11 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, if (module == NULL) return NULL; global = getattribute(module, global_name, self->proto >= 4); + Py_DECREF(module); } else { global = getattribute(module, global_name, self->proto >= 4); } - Py_DECREF(module); return global; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index c8a01d4e088..d9cfa3e2085 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1643,6 +1643,7 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; + PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; PyObject *rev_codes_dict; @@ -1692,6 +1693,11 @@ MODULE_INITFUNC(void) */ PyModule_AddStringConstant(m, "native_encoding", "UTF-8"); + sys_modules = PySys_GetObject("modules"); + if (sys_modules == NULL) { + Py_DECREF(m); + return NULL; + } d = PyModule_GetDict(m); if (d == NULL) { Py_DECREF(m); @@ -1701,7 +1707,7 @@ MODULE_INITFUNC(void) if (errors_module == NULL) { errors_module = PyModule_New(MODULE_NAME ".errors"); if (errors_module != NULL) { - _PyImport_SetModule(errmod_name, errors_module); + PyDict_SetItem(sys_modules, errmod_name, errors_module); /* gives away the reference to errors_module */ PyModule_AddObject(m, "errors", errors_module); } @@ -1711,7 +1717,7 @@ MODULE_INITFUNC(void) if (model_module == NULL) { model_module = PyModule_New(MODULE_NAME ".model"); if (model_module != NULL) { - _PyImport_SetModule(modelmod_name, model_module); + PyDict_SetItem(sys_modules, modelmod_name, model_module); /* gives away the reference to model_module */ PyModule_AddObject(m, "model", model_module); } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 2be49fbda38..5fab06d31df 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -161,18 +161,12 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) -{ - if (!_PyImport_IsInitialized(PyThreadState_GET()->interp)) - Py_FatalError("Python import machinery not initialized"); - return _PyModule_CreateInitialized(module, module_api_version); -} - -PyObject * -_PyModule_CreateInitialized(struct PyModuleDef* module, int module_api_version) { const char* name; PyModuleObject *m; - + PyInterpreterState *interp = PyThreadState_Get()->interp; + if (interp->modules == NULL) + Py_FatalError("Python import machinery not initialized"); if (!PyModuleDef_Init(module)) return NULL; name = module->m_name; diff --git a/Objects/object.c b/Objects/object.c index ed8a62a163a..74893e38c71 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -29,23 +29,20 @@ _Py_GetRefTotal(void) return total; } -PyObject * -_PyDebug_XOptionShowRefCount(void) -{ - PyObject *xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return NULL; - - _Py_IDENTIFIER(showrefcount); - return _PyDict_GetItemId(xoptions, &PyId_showrefcount); -} - void _PyDebug_PrintTotalRefs(void) { - fprintf(stderr, - "[%" PY_FORMAT_SIZE_T "d refs, " - "%" PY_FORMAT_SIZE_T "d blocks]\n", - _Py_GetRefTotal(), _Py_GetAllocatedBlocks()); + PyObject *xoptions, *value; + _Py_IDENTIFIER(showrefcount); + + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return; + value = _PyDict_GetItemId(xoptions, &PyId_showrefcount); + if (value == Py_True) + fprintf(stderr, + "[%" PY_FORMAT_SIZE_T "d refs, " + "%" PY_FORMAT_SIZE_T "d blocks]\n", + _Py_GetRefTotal(), _Py_GetAllocatedBlocks()); } #endif /* Py_REF_DEBUG */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a06cab72e5c..dc4d2edc5cf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3902,6 +3902,7 @@ import_copyreg(void) { PyObject *copyreg_str; PyObject *copyreg_module; + PyInterpreterState *interp = PyThreadState_GET()->interp; _Py_IDENTIFIER(copyreg); copyreg_str = _PyUnicode_FromId(&PyId_copyreg); @@ -3913,7 +3914,7 @@ import_copyreg(void) by storing a reference to the cached module in a static variable, but this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ - copyreg_module = _PyImport_GetModuleWithError(copyreg_str); + copyreg_module = PyDict_GetItemWithError(interp->modules, copyreg_str); if (copyreg_module != NULL) { Py_INCREF(copyreg_module); return copyreg_module; diff --git a/Python/_warnings.c b/Python/_warnings.c index 6dad0806867..ba004859df8 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -38,6 +38,7 @@ static PyObject * get_warnings_attr(const char *attr, int try_import) { static PyObject *warnings_str = NULL; + PyObject *all_modules; PyObject *warnings_module, *obj; if (warnings_str == NULL) { @@ -57,7 +58,9 @@ get_warnings_attr(const char *attr, int try_import) } } else { - warnings_module = _PyImport_GetModule(warnings_str); + all_modules = PyImport_GetModuleDict(); + + warnings_module = PyDict_GetItem(all_modules, warnings_str); if (warnings_module == NULL) return NULL; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c363cfe8cea..5e1f1d3854f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void) PyType_Ready(&PyZip_Type) < 0) return NULL; - mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); + mod = PyModule_Create(&builtinsmodule); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); diff --git a/Python/ceval.c b/Python/ceval.c index 5dd7cd9f03e..08533a42bf2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4935,7 +4935,7 @@ import_from(PyObject *v, PyObject *name) Py_DECREF(pkgname); return NULL; } - x = _PyImport_GetModule(fullmodname); + x = PyDict_GetItem(PyImport_GetModuleDict(), fullmodname); Py_DECREF(fullmodname); if (x == NULL) { goto error; diff --git a/Python/import.c b/Python/import.c index 6b2634c3497..2aacf659351 100644 --- a/Python/import.c +++ b/Python/import.c @@ -290,115 +290,10 @@ _PyImport_Fini(void) PyObject * PyImport_GetModuleDict(void) { - PyObject *sysdict = PyThreadState_GET()->interp->sysdict; - if (sysdict == NULL) { - Py_FatalError("PyImport_GetModuleDict: no sys module!"); - } - - _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); - if (modules == NULL) { - Py_FatalError("lost sys.modules"); - } - return modules; -} - -/* In some corner cases it is important to be sure that the import - machinery has been initialized (or not cleaned up yet). For - example, see issue #4236 and PyModule_Create2(). */ - -int -_PyImport_IsInitialized(PyInterpreterState *interp) -{ - if (interp->sysdict == NULL) - return 0; - _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); - if (modules == NULL) - return 0; - return 1; -} - -PyObject * -_PyImport_GetModule(PyObject *name) -{ - PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_CheckExact(modules)) { - return PyDict_GetItem(modules, name); - } - - PyObject *mod = PyObject_GetItem(modules, name); - // For backward-comaptibility we copy the behavior of PyDict_GetItem(). - if (PyErr_Occurred()) { - PyErr_Clear(); - } - Py_XDECREF(mod); - return mod; -} - -PyObject * -_PyImport_GetModuleWithError(PyObject *name) -{ - PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_CheckExact(modules)) { - return PyDict_GetItemWithError(modules, name); - } - - PyObject *mod = PyObject_GetItem(modules, name); - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - return mod; -} - -PyObject * -_PyImport_GetModuleId(struct _Py_Identifier *nameid) -{ - PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */ - if (name == NULL) { - return NULL; - } - return _PyImport_GetModule(name); -} - -int -_PyImport_SetModule(PyObject *name, PyObject *m) -{ - PyObject *modules = PyImport_GetModuleDict(); - return PyObject_SetItem(modules, name, m); -} - -int -_PyImport_SetModuleString(const char *name, PyObject *m) -{ - PyObject *modules = PyImport_GetModuleDict(); - return PyMapping_SetItemString(modules, name, m); -} - -PyObject * -PyImport_GetModule(PyObject *name) -{ - PyObject *m; - PyObject *modules = PyImport_GetModuleDict(); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules"); - return NULL; - } - Py_INCREF(modules); - if (PyDict_CheckExact(modules)) { - m = PyDict_GetItemWithError(modules, name); /* borrowed */ - Py_XINCREF(m); - } - else { - m = PyObject_GetItem(modules, name); - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - } - Py_DECREF(modules); - return m; + PyInterpreterState *interp = PyThreadState_GET()->interp; + if (interp->modules == NULL) + Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); + return interp->modules; } @@ -428,7 +323,7 @@ PyImport_Cleanup(void) Py_ssize_t pos; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; - PyObject *modules = PyImport_GetModuleDict(); + PyObject *modules = interp->modules; PyObject *weaklist = NULL; const char * const *p; @@ -490,7 +385,7 @@ PyImport_Cleanup(void) if (Py_VerboseFlag && PyUnicode_Check(key)) PySys_FormatStderr("# cleanup[2] removing %U\n", key); STORE_MODULE_WEAKREF(key, value); - PyObject_SetItem(modules, key, Py_None); + PyDict_SetItem(modules, key, Py_None); } } @@ -557,6 +452,7 @@ PyImport_Cleanup(void) /* Clear and delete the modules directory. Actual modules will still be there only if imported during the execution of some destructor. */ + interp->modules = NULL; Py_DECREF(modules); /* Once more */ @@ -615,9 +511,9 @@ PyImport_GetMagicTag(void) int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename, PyObject *modules) + PyObject *filename) { - PyObject *dict, *key; + PyObject *modules, *dict, *key; struct PyModuleDef *def; int res; if (extensions == NULL) { @@ -634,10 +530,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyErr_BadInternalCall(); return -1; } - if (PyObject_SetItem(modules, name, mod) < 0) + modules = PyImport_GetModuleDict(); + if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyMapping_DelItem(modules, name); + PyDict_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -665,28 +562,20 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } int -_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) +_PyImport_FixupBuiltin(PyObject *mod, const char *name) { int res; PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); Py_DECREF(nameobj); return res; } PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) -{ - PyObject *modules = PyImport_GetModuleDict(); - return _PyImport_FindExtensionObjectEx(name, filename, modules); -} - -PyObject * -_PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, - PyObject *modules) { PyObject *mod, *mdict, *key; PyModuleDef* def; @@ -703,7 +592,7 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = _PyImport_AddModuleObject(name, modules); + mod = PyImport_AddModuleObject(name); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -718,14 +607,14 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, mod = def->m_base.m_init(); if (mod == NULL) return NULL; - if (PyObject_SetItem(modules, name, mod) == -1) { + if (PyDict_SetItem(PyImport_GetModuleDict(), name, mod) == -1) { Py_DECREF(mod); return NULL; } Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyMapping_DelItem(modules, name); + PyDict_DelItem(PyImport_GetModuleDict(), name); Py_DECREF(mod); return NULL; } @@ -737,13 +626,13 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, } PyObject * -_PyImport_FindBuiltin(const char *name, PyObject *modules) +_PyImport_FindBuiltin(const char *name) { PyObject *res, *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionObjectEx(nameobj, nameobj, modules); + res = _PyImport_FindExtensionObject(nameobj, nameobj); Py_DECREF(nameobj); return res; } @@ -758,34 +647,19 @@ PyObject * PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return _PyImport_AddModuleObject(name, modules); -} - -PyObject * -_PyImport_AddModuleObject(PyObject *name, PyObject *modules) -{ PyObject *m; - if (PyDict_CheckExact(modules)) { - m = PyDict_GetItemWithError(modules, name); - } - else { - m = PyObject_GetItem(modules, name); - // For backward-comaptibility we copy the behavior - // of PyDict_GetItemWithError(). - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } + + if ((m = PyDict_GetItemWithError(modules, name)) != NULL && + PyModule_Check(m)) { + return m; } if (PyErr_Occurred()) { return NULL; } - if (m != NULL && PyModule_Check(m)) { - return m; - } m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyObject_SetItem(modules, name, m) != 0) { + if (PyDict_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -812,13 +686,11 @@ static void remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyMapping_DelItem(modules, name) < 0) { - if (!PyMapping_HasKey(modules, name)) { - return; - } + if (PyDict_GetItem(modules, name) == NULL) + return; + if (PyDict_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); - } } @@ -927,6 +799,7 @@ module_dict_for_exec(PyObject *name) static PyObject * exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object) { + PyObject *modules = PyImport_GetModuleDict(); PyObject *v, *m; v = PyEval_EvalCode(code_object, module_dict, module_dict); @@ -936,8 +809,7 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object } Py_DECREF(v); - m = _PyImport_GetModule(name); - if (m == NULL) { + if ((m = PyDict_GetItem(modules, name)) == NULL) { PyErr_Format(PyExc_ImportError, "Loaded module %R not found in sys.modules", name); @@ -1170,7 +1042,6 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *modules = NULL; for (p = PyImport_Inittab; p->name != NULL; p++) { PyModuleDef *def; if (_PyUnicode_EqualToASCIIString(name, p->name)) { @@ -1196,11 +1067,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } def->m_base.m_init = p->initfunc; - if (modules == NULL) { - modules = PyImport_GetModuleDict(); - } - if (_PyImport_FixupExtensionObject(mod, name, name, - modules) < 0) { + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) { Py_DECREF(name); return NULL; } @@ -1644,7 +1511,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, Py_INCREF(abs_name); } - mod = _PyImport_GetModule(abs_name); + mod = PyDict_GetItem(interp->modules, abs_name); if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); @@ -1731,7 +1598,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - final_mod = _PyImport_GetModule(to_return); + final_mod = PyDict_GetItem(interp->modules, to_return); Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, @@ -1784,10 +1651,10 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals PyObject * PyImport_ReloadModule(PyObject *m) { - _Py_IDENTIFIER(imp); _Py_IDENTIFIER(reload); PyObject *reloaded_module = NULL; - PyObject *imp = _PyImport_GetModuleId(&PyId_imp); + PyObject *modules = PyImport_GetModuleDict(); + PyObject *imp = PyDict_GetItemString(modules, "imp"); if (imp == NULL) { imp = PyImport_ImportModule("imp"); if (imp == NULL) { @@ -1822,6 +1689,7 @@ PyImport_Import(PyObject *module_name) PyObject *globals = NULL; PyObject *import = NULL; PyObject *builtins = NULL; + PyObject *modules = NULL; PyObject *r = NULL; /* Initialize constant string objects */ @@ -1876,7 +1744,8 @@ PyImport_Import(PyObject *module_name) goto err; Py_DECREF(r); - r = _PyImport_GetModule(module_name); + modules = PyImport_GetModuleDict(); + r = PyDict_GetItemWithError(modules, module_name); if (r != NULL) { Py_INCREF(r); } diff --git a/Python/importdl.c b/Python/importdl.c index 32fb7e1be21..d8656b94333 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -215,8 +215,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) else Py_INCREF(path); - PyObject *modules = PyImport_GetModuleDict(); - if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) + if (_PyImport_FixupExtensionObject(m, name_unicode, path) < 0) goto error; Py_DECREF(name_unicode); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3265d701810..0700569ac5a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -42,7 +42,6 @@ _Py_IDENTIFIER(name); _Py_IDENTIFIER(stdin); _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); -_Py_IDENTIFIER(threading); #ifdef __cplusplus extern "C" { @@ -284,6 +283,7 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) { PyObject *importlib; PyObject *impmod; + PyObject *sys_modules; PyObject *value; /* Import _importlib through its frozen version, _frozen_importlib. */ @@ -314,7 +314,11 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) else if (Py_VerboseFlag) { PySys_FormatStderr("import _imp # builtin\n"); } - if (_PyImport_SetModuleString("_imp", impmod) < 0) { + sys_modules = PyImport_GetModuleDict(); + if (Py_VerboseFlag) { + PySys_FormatStderr("import sys # builtin\n"); + } + if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) { Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); } @@ -671,20 +675,10 @@ void _Py_InitializeCore(const _PyCoreConfig *config) if (!_PyFloat_Init()) Py_FatalError("Py_InitializeCore: can't init float"); - PyObject *modules = PyDict_New(); - if (modules == NULL) + interp->modules = PyDict_New(); + if (interp->modules == NULL) Py_FatalError("Py_InitializeCore: can't make modules dictionary"); - sysmod = _PySys_BeginInit(); - if (sysmod == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys"); - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys dict"); - Py_INCREF(interp->sysdict); - PyDict_SetItemString(interp->sysdict, "modules", modules); - _PyImport_FixupBuiltin(sysmod, "sys", modules); - /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) Py_FatalError("Py_InitializeCore: can't initialize unicode"); @@ -695,7 +689,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config) bimod = _PyBuiltin_Init(); if (bimod == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins modules"); - _PyImport_FixupBuiltin(bimod, "builtins", modules); + _PyImport_FixupBuiltin(bimod, "builtins"); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins dict"); @@ -704,6 +698,17 @@ void _Py_InitializeCore(const _PyCoreConfig *config) /* initialize builtin exceptions */ _PyExc_Init(bimod); + sysmod = _PySys_BeginInit(); + if (sysmod == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys"); + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys dict"); + Py_INCREF(interp->sysdict); + _PyImport_FixupBuiltin(sysmod, "sys"); + PyDict_SetItemString(interp->sysdict, "modules", + interp->modules); + /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); @@ -1006,11 +1011,6 @@ Py_FinalizeEx(void) while (_PyGC_CollectIfEnabled() > 0) /* nothing */; #endif - -#ifdef Py_REF_DEBUG - PyObject *showrefcount = _PyDebug_XOptionShowRefCount(); -#endif - /* Destroy all modules */ PyImport_Cleanup(); @@ -1058,10 +1058,7 @@ Py_FinalizeEx(void) /* dump hash stats */ _PyHash_Fini(); -#ifdef Py_REF_DEBUG - if (showrefcount == Py_True) - _PyDebug_PrintTotalRefs(); -#endif + _PY_DEBUG_PRINT_TOTAL_REFS(); #ifdef Py_TRACE_REFS /* Display all objects still alive -- this can invoke arbitrary @@ -1206,22 +1203,9 @@ Py_NewInterpreter(void) /* XXX The following is lax in error checking */ - PyObject *modules = PyDict_New(); - if (modules == NULL) - Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); + interp->modules = PyDict_New(); - sysmod = _PyImport_FindBuiltin("sys", modules); - if (sysmod != NULL) { - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - goto handle_error; - Py_INCREF(interp->sysdict); - PyDict_SetItemString(interp->sysdict, "modules", modules); - PySys_SetPath(Py_GetPath()); - _PySys_EndInit(interp->sysdict); - } - - bimod = _PyImport_FindBuiltin("builtins", modules); + bimod = _PyImport_FindBuiltin("builtins"); if (bimod != NULL) { interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) @@ -1232,9 +1216,18 @@ Py_NewInterpreter(void) /* initialize builtin exceptions */ _PyExc_Init(bimod); + sysmod = _PyImport_FindBuiltin("sys"); if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + goto handle_error; + Py_INCREF(interp->sysdict); + _PySys_EndInit(interp->sysdict); + PySys_SetPath(Py_GetPath()); + PyDict_SetItemString(interp->sysdict, "modules", + interp->modules); /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); @@ -1910,13 +1903,14 @@ wait_for_thread_shutdown(void) { _Py_IDENTIFIER(_shutdown); PyObject *result; - PyObject *threading = _PyImport_GetModuleId(&PyId_threading); + PyThreadState *tstate = PyThreadState_GET(); + PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, + "threading"); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); return; } - Py_INCREF(threading); result = _PyObject_CallMethodId(threading, &PyId__shutdown, NULL); if (result == NULL) { PyErr_WriteUnraisable(threading); diff --git a/Python/pystate.c b/Python/pystate.c index 08048428610..41adf03662c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -90,12 +90,15 @@ PyInterpreterState_New(void) PyMem_RawMalloc(sizeof(PyInterpreterState)); if (interp != NULL) { + interp->modules = NULL; interp->modules_by_index = NULL; interp->sysdict = NULL; interp->builtins = NULL; interp->builtins_copy = NULL; interp->tstate_head = NULL; interp->check_interval = 100; + interp->warnoptions = NULL; + interp->xoptions = NULL; interp->num_threads = 0; interp->pythread_stacksize = 0; interp->codec_search_path = NULL; @@ -153,6 +156,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp) Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); + Py_CLEAR(interp->modules); Py_CLEAR(interp->modules_by_index); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index df814fbed97..d1d4a69a8dd 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -113,10 +113,7 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags * err = -1; for (;;) { ret = PyRun_InteractiveOneObject(fp, filename, flags); -#ifdef Py_REF_DEBUG - if (_PyDebug_XOptionShowRefCount() == Py_True) - _PyDebug_PrintTotalRefs(); -#endif + _PY_DEBUG_PRINT_TOTAL_REFS(); if (ret == E_EOF) { err = 0; break; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5bde4b7b702..dd127a1f7ce 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -36,14 +36,12 @@ extern const char *PyWin_DLLVersionString; _Py_IDENTIFIER(_); _Py_IDENTIFIER(__sizeof__); -_Py_IDENTIFIER(_xoptions); _Py_IDENTIFIER(buffer); _Py_IDENTIFIER(builtins); _Py_IDENTIFIER(encoding); _Py_IDENTIFIER(path); _Py_IDENTIFIER(stdout); _Py_IDENTIFIER(stderr); -_Py_IDENTIFIER(warnoptions); _Py_IDENTIFIER(write); PyObject * @@ -162,11 +160,13 @@ static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; + PyInterpreterState *interp = PyThreadState_GET()->interp; + PyObject *modules = interp->modules; PyObject *builtins; static PyObject *newline = NULL; int err; - builtins = _PyImport_GetModuleId(&PyId_builtins); + builtins = _PyDict_GetItemId(modules, &PyId_builtins); if (builtins == NULL) { PyErr_SetString(PyExc_RuntimeError, "lost builtins module"); return NULL; @@ -1481,17 +1481,13 @@ list_builtin_module_names(void) static PyObject * get_warnoptions(void) { - PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); + PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions; if (warnoptions == NULL || !PyList_Check(warnoptions)) { Py_XDECREF(warnoptions); warnoptions = PyList_New(0); if (warnoptions == NULL) return NULL; - if (_PySys_SetObjectId(&PyId_warnoptions, warnoptions)) { - Py_DECREF(warnoptions); - return NULL; - } - Py_DECREF(warnoptions); + PyThreadState_GET()->interp->warnoptions = warnoptions; } return warnoptions; } @@ -1499,7 +1495,7 @@ get_warnoptions(void) void PySys_ResetWarnOptions(void) { - PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); + PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions; if (warnoptions == NULL || !PyList_Check(warnoptions)) return; PyList_SetSlice(warnoptions, 0, PyList_GET_SIZE(warnoptions), NULL); @@ -1528,24 +1524,20 @@ PySys_AddWarnOption(const wchar_t *s) int PySys_HasWarnOptions(void) { - PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions); + PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions; return (warnoptions != NULL && (PyList_Size(warnoptions) > 0)) ? 1 : 0; } static PyObject * get_xoptions(void) { - PyObject *xoptions = _PySys_GetObjectId(&PyId__xoptions); + PyObject *xoptions = PyThreadState_GET()->interp->xoptions; if (xoptions == NULL || !PyDict_Check(xoptions)) { Py_XDECREF(xoptions); xoptions = PyDict_New(); if (xoptions == NULL) return NULL; - if (_PySys_SetObjectId(&PyId__xoptions, xoptions)) { - Py_DECREF(xoptions); - return NULL; - } - Py_DECREF(xoptions); + PyThreadState_GET()->interp->xoptions = xoptions; } return xoptions; } @@ -1947,7 +1939,7 @@ _PySys_BeginInit(void) PyObject *m, *sysdict, *version_info; int res; - m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); + m = PyModule_Create(&sysmodule); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); @@ -2094,6 +2086,16 @@ _PySys_BeginInit(void) #undef SET_SYS_FROM_STRING_BORROW /* Updating the sys namespace, returning integer error codes */ +#define SET_SYS_FROM_STRING_BORROW_INT_RESULT(key, value) \ + do { \ + PyObject *v = (value); \ + if (v == NULL) \ + return -1; \ + res = PyDict_SetItemString(sysdict, key, v); \ + if (res < 0) { \ + return res; \ + } \ + } while (0) #define SET_SYS_FROM_STRING_INT_RESULT(key, value) \ do { \ PyObject *v = (value); \ @@ -2138,11 +2140,15 @@ _PySys_EndInit(PyObject *sysdict) SET_SYS_FROM_STRING_INT_RESULT("base_exec_prefix", PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); - if (get_warnoptions() == NULL) + PyObject *warnoptions = get_warnoptions(); + if (warnoptions == NULL) return -1; + SET_SYS_FROM_STRING_BORROW_INT_RESULT("warnoptions", warnoptions); - if (get_xoptions() == NULL) + PyObject *xoptions = get_xoptions(); + if (xoptions == NULL) return -1; + SET_SYS_FROM_STRING_BORROW_INT_RESULT("_xoptions", xoptions); if (PyErr_Occurred()) return -1; @@ -2150,6 +2156,7 @@ _PySys_EndInit(PyObject *sysdict) } #undef SET_SYS_FROM_STRING_INT_RESULT +#undef SET_SYS_FROM_STRING_BORROW_INT_RESULT static PyObject * makepathobject(const wchar_t *path, wchar_t delim)