From f341d6017dd4e80509b69b5a9e2625b71b70f205 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:35:52 -0600 Subject: [PATCH] gh-76785: Add PyInterpreterConfig Helpers (gh-117170) These helpers make it easier to customize and inspect the config used to initialize interpreters. This is especially valuable in our tests. I found inspiration from the PyConfig API for the PyInterpreterConfig dict conversion stuff. As part of this PR I've also added a bunch of tests. --- Include/internal/pycore_pylifecycle.h | 16 ++ Lib/test/support/__init__.py | 15 +- Lib/test/test_capi/test_misc.py | 251 +++++++++++++++++++++++ Lib/test/test_import/__init__.py | 12 +- Makefile.pre.in | 5 + Modules/_testinternalcapi.c | 226 ++++++++++++++------- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/config_common.h | 36 ++++ Python/initconfig.c | 25 +-- Python/interpconfig.c | 266 +++++++++++++++++++++++++ 13 files changed, 764 insertions(+), 96 deletions(-) create mode 100644 Python/config_common.h create mode 100644 Python/interpconfig.c diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c6750986857..47ff0806574 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -116,6 +116,22 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 92e3174407f..9640d5d831b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1734,8 +1734,19 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + else: + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 55a1ab6d6d9..34311afc93f 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,257 @@ class SubinterpreterTest(unittest.TestCase): self.assertEqual(main_attr_id, subinterp_attr_id) +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _testinternalcapi.new_interp_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(gil=value) + + @requires_subinterpreters + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(RuntimeError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + @requires_subinterpreters + def test_get_config(self): + @contextlib.contextmanager + def new_interp(config): + interpid = _testinternalcapi.new_interpreter(config) + try: + yield interpid + finally: + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + + with self.subTest('main'): + expected = _testinternalcapi.new_interp_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _testinternalcapi.new_interp_config('isolated') + with new_interp('isolated') as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _testinternalcapi.new_interp_config('legacy') + with new_interp('legacy') as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _testinternalcapi.new_interp_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + ) + with new_interp(orig) as interpid: + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_subinterpreters class InterpreterIDTests(unittest.TestCase): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4deed7f3ba2..81ec700d975 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1823,15 +1823,19 @@ class SubinterpImportTests(unittest.TestCase): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=strict, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1847,12 +1851,16 @@ class SubinterpImportTests(unittest.TestCase): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=True, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) diff --git a/Makefile.pre.in b/Makefile.pre.in index f5c2af0696a..2a22a1e95a3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -440,6 +440,7 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ Python/jit.o \ @@ -1687,6 +1688,10 @@ Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $ Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d6d50e75b61..56761d1a896 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,10 +23,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1355,83 +1357,153 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static int +init_named_interp_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL) { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static PyObject * +new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { + return NULL; + } + PyObject *overrides = kwds; + + if (name == NULL) { + name = "isolated"; + } + + PyInterpreterConfig config; + if (init_named_interp_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static int +interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_interp_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; - - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; + PyObject *configobj; + static char *kwlist[] = {"code", "config", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { - return NULL; - } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); - return NULL; - } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); - return NULL; - } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); - return NULL; - } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); - return NULL; - } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); - return NULL; - } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); - return NULL; - } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + "sO:run_in_subinterp_with_config", kwlist, + &code, &configobj)) + { return NULL; } - mainstate = PyThreadState_Get(); + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { + return NULL; + } + + PyThreadState *mainstate = PyThreadState_Get(); PyThreadState_Swap(NULL); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; + PyThreadState *substate; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to @@ -1445,7 +1517,9 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); @@ -1473,13 +1547,21 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -new_interpreter(PyObject *self, PyObject *Py_UNUSED(ignored)) +new_interpreter(PyObject *self, PyObject *args) { + PyObject *configobj = NULL; + if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { + return NULL; + } + + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { + return NULL; + } + // Unlike _interpreters.create(), we do not automatically link // the interpreter to its refcount. PyThreadState *save_tstate = PyThreadState_Get(); - const PyInterpreterConfig config = \ - (PyInterpreterConfig)_PyInterpreterConfig_INIT; PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); PyThreadState_Swap(save_tstate); @@ -1846,12 +1928,16 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"new_interp_config", _PyCFunction_CAST(new_interp_config), + METH_VARARGS | METH_KEYWORDS}, + {"get_interp_config", _PyCFunction_CAST(get_interp_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_NOARGS}, + {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 82471e0f140..9c82fcf021b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 97c52fdadf7..63b033a0350 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -229,6 +229,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7a2a98df651..657ffd1aa4c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -587,6 +587,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 89b56ec1267..6e0cd1754f5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1343,6 +1343,9 @@ Python + + Python + Source Files diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000..e749bd4bf0d --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/initconfig.c b/Python/initconfig.c index 215d6a1d4e0..d91a8199b54 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,9 @@ # endif #endif +#include "config_common.h" + + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { @@ -1098,21 +1101,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; - } - if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1120,13 +1108,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000..419f40ae62a --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,266 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(STR); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + Py_DECREF(item); + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +}