From cdad2724e6f7426372901cc5dedd8a462ba046a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Apr 2021 00:52:52 +0200 Subject: [PATCH] bpo-40137: Add pycore_moduleobject.h internal header (GH-25507) Add pycore_moduleobject.h internal header file with static inline functions to access module members: * _PyModule_GetDict() * _PyModule_GetDef() * _PyModule_GetState() These functions don't check at runtime if their argument has a valid type and can be inlined even if Python is not built with LTO. _PyType_GetModuleByDef() uses _PyModule_GetDef(). Replace PyModule_GetState() with _PyModule_GetState() in the extension modules, considered as performance sensitive: * _abc * _functools * _operator * _pickle * _queue * _random * _sre * _struct * _thread * _winapi * array * posix The following extensions are now built with the Py_BUILD_CORE_MODULE macro defined, to be able to use the internal pycore_moduleobject.h header: _abc, array, _operator, _queue, _sre, _struct. --- Include/internal/pycore_moduleobject.h | 42 ++++++++++++++++++++++++++ Makefile.pre.in | 1 + Modules/Setup | 12 ++++---- Modules/_abc.c | 3 +- Modules/_functoolsmodule.c | 6 ++-- Modules/_operator.c | 5 ++- Modules/_pickle.c | 3 +- Modules/_queuemodule.c | 3 +- Modules/_randommodule.c | 5 +-- Modules/_sre.c | 3 +- Modules/_struct.c | 3 +- Modules/_threadmodule.c | 5 +-- Modules/_winapi.c | 4 +-- Modules/arraymodule.c | 3 +- Modules/posixmodule.c | 3 +- Objects/frameobject.c | 3 +- Objects/moduleobject.c | 19 +++--------- Objects/typeobject.c | 5 +-- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ setup.py | 12 +++++--- 21 files changed, 97 insertions(+), 47 deletions(-) create mode 100644 Include/internal/pycore_moduleobject.h diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h new file mode 100644 index 00000000000..e9978abd25b --- /dev/null +++ b/Include/internal/pycore_moduleobject.h @@ -0,0 +1,42 @@ +#ifndef Py_INTERNAL_MODULEOBJECT_H +#define Py_INTERNAL_MODULEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +typedef struct { + PyObject_HEAD + PyObject *md_dict; + struct PyModuleDef *md_def; + void *md_state; + PyObject *md_weaklist; + // for logging purposes after md_dict is cleared + PyObject *md_name; +} PyModuleObject; + +static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { + assert(PyModule_Check(mod)); + return ((PyModuleObject *)mod)->md_def; +} + +static inline void* _PyModule_GetState(PyObject* mod) { + assert(PyModule_Check(mod)); + return ((PyModuleObject *)mod)->md_state; +} + +static inline PyObject* _PyModule_GetDict(PyObject *mod) { + assert(PyModule_Check(mod)); + PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) + assert(dict != NULL); + return dict; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MODULEOBJECT_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index eccc7269770..4d9b760863a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1161,6 +1161,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_interp.h \ $(srcdir)/Include/internal/pycore_list.h \ $(srcdir)/Include/internal/pycore_long.h \ + $(srcdir)/Include/internal/pycore_moduleobject.h \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ diff --git a/Modules/Setup b/Modules/Setup index cce78582a1e..87c6a152f86 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -105,13 +105,13 @@ posix -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal posixmodule.c # posix errno errnomodule.c # posix (UNIX) errno values pwd pwdmodule.c # this is needed to find out the user's home dir # if $HOME is not set -_sre _sre.c # Fredrik Lundh's new regular expressions +_sre -DPy_BUILD_CORE_BUILTIN _sre.c # Fredrik Lundh's new regular expressions _codecs _codecsmodule.c # access to the builtin codecs and codec registry _weakref _weakref.c # weak references _functools -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal _functoolsmodule.c # Tools for working with functions and callable objects -_operator _operator.c # operator.add() and similar goodies +_operator -DPy_BUILD_CORE_BUILTIN _operator.c # operator.add() and similar goodies _collections _collectionsmodule.c # Container types -_abc _abc.c # Abstract base classes +_abc -DPy_BUILD_CORE_BUILTIN _abc.c # Abstract base classes itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _signal -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal signalmodule.c @@ -166,17 +166,17 @@ _symtable symtablemodule.c # Modules that should always be present (non UNIX dependent): -#array arraymodule.c # array objects +#array -DPy_BUILD_CORE_MODULE arraymodule.c # array objects #cmath cmathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # complex math library functions #math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin() #_contextvars _contextvarsmodule.c # Context Variables -#_struct _struct.c # binary structure packing/unpacking +#_struct -DPy_BUILD_CORE_MODULE _struct.c # binary structure packing/unpacking #_weakref _weakref.c # basic weak reference support #_testcapi _testcapimodule.c # Python C API test module #_testinternalcapi _testinternalcapi.c -I$(srcdir)/Include/internal -DPy_BUILD_CORE_MODULE # Python internal C API test module #_random _randommodule.c -DPy_BUILD_CORE_MODULE # Random number generator #_elementtree -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI _elementtree.c # elementtree accelerator -#_pickle _pickle.c # pickle accelerator +#_pickle -DPy_BUILD_CORE_MODULE _pickle.c # pickle accelerator #_datetime _datetimemodule.c # datetime accelerator #_zoneinfo _zoneinfo.c -DPy_BUILD_CORE_MODULE # zoneinfo accelerator #_bisect _bisectmodule.c # Bisection algorithms diff --git a/Modules/_abc.c b/Modules/_abc.c index 7afaa759b2b..0ddc2abeee1 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -1,6 +1,7 @@ /* ABCMeta implementation */ #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "clinic/_abc.c.h" /*[clinic input] @@ -27,7 +28,7 @@ typedef struct { static inline _abcmodule_state* get_abc_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_abcmodule_state *)state; } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index f3ae3b62ae3..eea542e18c9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -23,7 +24,7 @@ typedef struct _functools_state { static inline _functools_state * get_functools_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_functools_state *)state; } @@ -53,8 +54,7 @@ get_functools_state_by_type(PyTypeObject *type) if (module == NULL) { return NULL; } - _functools_state *state = get_functools_state(module); - return state; + return get_functools_state(module); } static PyObject * diff --git a/Modules/_operator.c b/Modules/_operator.c index da1e43158ee..f55c2f1d1f4 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1,6 +1,5 @@ - #include "Python.h" - +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "clinic/_operator.c.h" typedef struct { @@ -12,7 +11,7 @@ typedef struct { static inline _operator_state* get_operator_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_operator_state *)state; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 5a8aad9de76..691d4a293e8 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -9,6 +9,7 @@ #endif #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef PyDoc_STRVAR(pickle_module_doc, @@ -182,7 +183,7 @@ static struct PyModuleDef _picklemodule; static PickleState * _Pickle_GetState(PyObject *module) { - return (PickleState *)PyModule_GetState(module); + return (PickleState *)_PyModule_GetState(module); } /* Find the module instance imported in the currently running sub-interpreter diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 7a52617ade5..c27fb1a001d 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include // offsetof() @@ -10,7 +11,7 @@ typedef struct { static simplequeue_state * simplequeue_get_state(PyObject *module) { - simplequeue_state *state = PyModule_GetState(module); + simplequeue_state *state = _PyModule_GetState(module); assert(state); return state; } diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 99be69c0655..cae49a009cb 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -67,6 +67,7 @@ /* ---------------------------------------------------------------*/ #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #ifdef HAVE_PROCESS_H # include // getpid() #endif @@ -86,7 +87,7 @@ typedef struct { static inline _randomstate* get_random_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_randomstate *)state; } @@ -538,7 +539,7 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyTuple_GET_SIZE(args) == 1) arg = PyTuple_GET_ITEM(args, 0); - + tmp = random_seed(self, arg); if (tmp == NULL) { Py_DECREF(self); diff --git a/Modules/_sre.c b/Modules/_sre.c index 57faf7bdaae..d4bfff6e849 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -42,6 +42,7 @@ static const char copyright[] = #include "Python.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include "sre.h" @@ -258,7 +259,7 @@ typedef struct { static _sremodulestate * get_sre_module_state(PyObject *m) { - _sremodulestate *state = (_sremodulestate *)PyModule_GetState(m); + _sremodulestate *state = (_sremodulestate *)_PyModule_GetState(m); assert(state); return state; } diff --git a/Modules/_struct.c b/Modules/_struct.c index 1a5e0ae28e8..30ad9f2b79d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -6,6 +6,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include @@ -24,7 +25,7 @@ typedef struct { static inline _structmodulestate* get_struct_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_structmodulestate *)state; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0613dfd3070..7feb0b8a1f1 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -3,8 +3,9 @@ /* Interface to Sjoerd's portable C thread library */ #include "Python.h" -#include "pycore_pylifecycle.h" #include "pycore_interp.h" // _PyInterpreterState.num_threads +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_Init() #include // offsetof() #include "structmember.h" // PyMemberDef @@ -35,7 +36,7 @@ typedef struct { static inline thread_module_state* get_thread_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (thread_module_state *)state; } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 7ba14095c96..9d5a45adac5 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -35,7 +35,7 @@ /* See http://www.python.org/2.4/license for licensing details. */ #include "Python.h" -#include "moduleobject.h" // PyModuleDef_Slot +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef @@ -87,7 +87,7 @@ typedef struct { static inline WinApiState* winapi_get_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (WinApiState *)state; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index fb9ebbe9f48..f5326789521 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -5,6 +5,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef #include // offsetof() @@ -63,7 +64,7 @@ typedef struct { static array_state * get_array_state(PyObject *module) { - return (array_state *)PyModule_GetState(module); + return (array_state *)_PyModule_GetState(module); } #define find_array_state_by_type(tp) \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65e8d5e7bd9..8ce62c88216 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_fileutils.h" +#include "pycore_moduleobject.h" // _PyModule_GetState() #ifdef MS_WINDOWS /* include early to avoid conflict with pycore_condvar.h: @@ -994,7 +995,7 @@ typedef struct { static inline _posixstate* get_posix_state(PyObject *module) { - void *state = PyModule_GetState(module); + void *state = _PyModule_GetState(module); assert(state != NULL); return (_posixstate *)state; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 9687ba5c7a9..b0487c2b688 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals() +#include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "frameobject.h" // PyFrameObject @@ -1176,7 +1177,7 @@ _PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) PyObject *builtins = _PyDict_GetItemIdWithError(globals, &PyId___builtins__); if (builtins) { if (PyModule_Check(builtins)) { - builtins = PyModule_GetDict(builtins); + builtins = _PyModule_GetDict(builtins); assert(builtins != NULL); } return builtins; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index e57ea86e769..a6eb85bdc2a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_moduleobject.h" // _PyModule_GetDef() #include "structmember.h" // PyMemberDef static Py_ssize_t max_module_number; @@ -12,15 +13,6 @@ _Py_IDENTIFIER(__doc__); _Py_IDENTIFIER(__name__); _Py_IDENTIFIER(__spec__); -typedef struct { - PyObject_HEAD - PyObject *md_dict; - struct PyModuleDef *md_def; - void *md_state; - PyObject *md_weaklist; - PyObject *md_name; /* for logging purposes after md_dict is cleared */ -} PyModuleObject; - static PyMemberDef module_members[] = { {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY}, {0} @@ -469,14 +461,11 @@ PyModule_SetDocString(PyObject *m, const char *doc) PyObject * PyModule_GetDict(PyObject *m) { - PyObject *d; if (!PyModule_Check(m)) { PyErr_BadInternalCall(); return NULL; } - d = ((PyModuleObject *)m) -> md_dict; - assert(d != NULL); - return d; + return _PyModule_GetDict(m); } PyObject* @@ -556,7 +545,7 @@ PyModule_GetDef(PyObject* m) PyErr_BadArgument(); return NULL; } - return ((PyModuleObject *)m)->md_def; + return _PyModule_GetDef(m); } void* @@ -566,7 +555,7 @@ PyModule_GetState(PyObject* m) PyErr_BadArgument(); return NULL; } - return ((PyModuleObject *)m)->md_state; + return _PyModule_GetState(m); } void diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 03af2c5d75d..254d12cc970 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4,6 +4,7 @@ #include "pycore_call.h" #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" +#include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() @@ -3582,7 +3583,7 @@ PyType_GetModuleState(PyTypeObject *type) if (m == NULL) { return NULL; } - return PyModule_GetState(m); + return _PyModule_GetState(m); } @@ -3617,7 +3618,7 @@ _PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; - if (module && PyModule_GetDef(module) == def) { + if (module && _PyModule_GetDef(module) == def) { return module; } i++; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3c4785c077e..429d437c5eb 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -199,6 +199,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 8c104bf7b31..55b57ef29dc 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -558,6 +558,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/setup.py b/setup.py index af384409553..df434d48768 100644 --- a/setup.py +++ b/setup.py @@ -869,7 +869,8 @@ class PyBuildExt(build_ext): # # array objects - self.add(Extension('array', ['arraymodule.c'])) + self.add(Extension('array', ['arraymodule.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # Context Variables self.add(Extension('_contextvars', ['_contextvarsmodule.c'])) @@ -934,9 +935,11 @@ class PyBuildExt(build_ext): self.add(Extension("_asyncio", ["_asynciomodule.c"], extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # _abc speedups - self.add(Extension("_abc", ["_abc.c"])) + self.add(Extension("_abc", ["_abc.c"], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # _queue module - self.add(Extension("_queue", ["_queuemodule.c"])) + self.add(Extension("_queue", ["_queuemodule.c"], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])) # _statistics module self.add(Extension("_statistics", ["_statisticsmodule.c"])) @@ -2696,7 +2699,8 @@ def main(): 'install_lib': PyBuildInstallLib}, # The struct module is defined here, because build_ext won't be # called unless there's at least one extension module defined. - ext_modules=[Extension('_struct', ['_struct.c'])], + ext_modules=[Extension('_struct', ['_struct.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'])], # If you change the scripts installed here, you also need to # check the PyBuildScripts command above, and change the links