From 5cf4782a2630629d0978bf4cf6b6340365f449b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Nov 2020 22:03:28 +0100 Subject: [PATCH] bpo-41796: Make _ast module state per interpreter (GH-23024) The ast module internal state is now per interpreter. * Rename "astmodulestate" to "struct ast_state" * Add pycore_ast.h internal header: the ast_state structure is now declared in pycore_ast.h. * Add PyInterpreterState.ast (struct ast_state) * Remove get_ast_state() * Rename get_global_ast_state() to get_ast_state() * PyAST_obj2mod() now handles get_ast_state() failures --- Include/Python-ast.h | 2 +- Include/internal/pycore_ast.h | 234 +++++++++++++++++ Include/internal/pycore_interp.h | 11 +- Makefile.pre.in | 2 + .../2020-10-29-12-49-08.bpo-41796.tkGdHq.rst | 2 + PCbuild/regen.vcxproj | 12 +- Parser/asdl_c.py | 243 +++++++++++------ Python/Python-ast.c | 245 ++++++++++-------- 8 files changed, 549 insertions(+), 202 deletions(-) create mode 100644 Include/internal/pycore_ast.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst diff --git a/Include/Python-ast.h b/Include/Python-ast.h index e14bab566fb..fc9f65c97a2 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -1,4 +1,4 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. #ifndef Py_PYTHON_AST_H #define Py_PYTHON_AST_H diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h new file mode 100644 index 00000000000..058fbc0fcdc --- /dev/null +++ b/Include/internal/pycore_ast.h @@ -0,0 +1,234 @@ +// File automatically generated by Parser/asdl_c.py. + +#ifndef Py_INTERNAL_AST_H +#define Py_INTERNAL_AST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct ast_state { + int initialized; + PyObject *AST_type; + PyObject *Add_singleton; + PyObject *Add_type; + PyObject *And_singleton; + PyObject *And_type; + PyObject *AnnAssign_type; + PyObject *Assert_type; + PyObject *Assign_type; + PyObject *AsyncFor_type; + PyObject *AsyncFunctionDef_type; + PyObject *AsyncWith_type; + PyObject *Attribute_type; + PyObject *AugAssign_type; + PyObject *Await_type; + PyObject *BinOp_type; + PyObject *BitAnd_singleton; + PyObject *BitAnd_type; + PyObject *BitOr_singleton; + PyObject *BitOr_type; + PyObject *BitXor_singleton; + PyObject *BitXor_type; + PyObject *BoolOp_type; + PyObject *Break_type; + PyObject *Call_type; + PyObject *ClassDef_type; + PyObject *Compare_type; + PyObject *Constant_type; + PyObject *Continue_type; + PyObject *Del_singleton; + PyObject *Del_type; + PyObject *Delete_type; + PyObject *DictComp_type; + PyObject *Dict_type; + PyObject *Div_singleton; + PyObject *Div_type; + PyObject *Eq_singleton; + PyObject *Eq_type; + PyObject *ExceptHandler_type; + PyObject *Expr_type; + PyObject *Expression_type; + PyObject *FloorDiv_singleton; + PyObject *FloorDiv_type; + PyObject *For_type; + PyObject *FormattedValue_type; + PyObject *FunctionDef_type; + PyObject *FunctionType_type; + PyObject *GeneratorExp_type; + PyObject *Global_type; + PyObject *GtE_singleton; + PyObject *GtE_type; + PyObject *Gt_singleton; + PyObject *Gt_type; + PyObject *IfExp_type; + PyObject *If_type; + PyObject *ImportFrom_type; + PyObject *Import_type; + PyObject *In_singleton; + PyObject *In_type; + PyObject *Interactive_type; + PyObject *Invert_singleton; + PyObject *Invert_type; + PyObject *IsNot_singleton; + PyObject *IsNot_type; + PyObject *Is_singleton; + PyObject *Is_type; + PyObject *JoinedStr_type; + PyObject *LShift_singleton; + PyObject *LShift_type; + PyObject *Lambda_type; + PyObject *ListComp_type; + PyObject *List_type; + PyObject *Load_singleton; + PyObject *Load_type; + PyObject *LtE_singleton; + PyObject *LtE_type; + PyObject *Lt_singleton; + PyObject *Lt_type; + PyObject *MatMult_singleton; + PyObject *MatMult_type; + PyObject *Mod_singleton; + PyObject *Mod_type; + PyObject *Module_type; + PyObject *Mult_singleton; + PyObject *Mult_type; + PyObject *Name_type; + PyObject *NamedExpr_type; + PyObject *Nonlocal_type; + PyObject *NotEq_singleton; + PyObject *NotEq_type; + PyObject *NotIn_singleton; + PyObject *NotIn_type; + PyObject *Not_singleton; + PyObject *Not_type; + PyObject *Or_singleton; + PyObject *Or_type; + PyObject *Pass_type; + PyObject *Pow_singleton; + PyObject *Pow_type; + PyObject *RShift_singleton; + PyObject *RShift_type; + PyObject *Raise_type; + PyObject *Return_type; + PyObject *SetComp_type; + PyObject *Set_type; + PyObject *Slice_type; + PyObject *Starred_type; + PyObject *Store_singleton; + PyObject *Store_type; + PyObject *Sub_singleton; + PyObject *Sub_type; + PyObject *Subscript_type; + PyObject *Try_type; + PyObject *Tuple_type; + PyObject *TypeIgnore_type; + PyObject *UAdd_singleton; + PyObject *UAdd_type; + PyObject *USub_singleton; + PyObject *USub_type; + PyObject *UnaryOp_type; + PyObject *While_type; + PyObject *With_type; + PyObject *YieldFrom_type; + PyObject *Yield_type; + PyObject *__dict__; + PyObject *__doc__; + PyObject *__module__; + PyObject *_attributes; + PyObject *_fields; + PyObject *alias_type; + PyObject *annotation; + PyObject *arg; + PyObject *arg_type; + PyObject *args; + PyObject *argtypes; + PyObject *arguments_type; + PyObject *asname; + PyObject *ast; + PyObject *attr; + PyObject *bases; + PyObject *body; + PyObject *boolop_type; + PyObject *cause; + PyObject *cmpop_type; + PyObject *col_offset; + PyObject *comparators; + PyObject *comprehension_type; + PyObject *context_expr; + PyObject *conversion; + PyObject *ctx; + PyObject *decorator_list; + PyObject *defaults; + PyObject *elt; + PyObject *elts; + PyObject *end_col_offset; + PyObject *end_lineno; + PyObject *exc; + PyObject *excepthandler_type; + PyObject *expr_context_type; + PyObject *expr_type; + PyObject *finalbody; + PyObject *format_spec; + PyObject *func; + PyObject *generators; + PyObject *handlers; + PyObject *id; + PyObject *ifs; + PyObject *is_async; + PyObject *items; + PyObject *iter; + PyObject *key; + PyObject *keys; + PyObject *keyword_type; + PyObject *keywords; + PyObject *kind; + PyObject *kw_defaults; + PyObject *kwarg; + PyObject *kwonlyargs; + PyObject *left; + PyObject *level; + PyObject *lineno; + PyObject *lower; + PyObject *mod_type; + PyObject *module; + PyObject *msg; + PyObject *name; + PyObject *names; + PyObject *op; + PyObject *operand; + PyObject *operator_type; + PyObject *ops; + PyObject *optional_vars; + PyObject *orelse; + PyObject *posonlyargs; + PyObject *returns; + PyObject *right; + PyObject *simple; + PyObject *slice; + PyObject *step; + PyObject *stmt_type; + PyObject *tag; + PyObject *target; + PyObject *targets; + PyObject *test; + PyObject *type; + PyObject *type_comment; + PyObject *type_ignore_type; + PyObject *type_ignores; + PyObject *unaryop_type; + PyObject *upper; + PyObject *value; + PyObject *values; + PyObject *vararg; + PyObject *withitem_type; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AST_H */ + diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 69d2108da43..9923b6b03da 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_gil.h" /* struct _gil_runtime_state */ -#include "pycore_gc.h" /* struct _gc_runtime_state */ -#include "pycore_warnings.h" /* struct _warnings_runtime_state */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_ast.h" // struct ast_state +#include "pycore_gil.h" // struct _gil_runtime_state +#include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { PyThread_type_lock lock; @@ -258,6 +259,8 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; + + struct ast_state ast; }; /* Used by _PyImport_Cleanup() */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5b6c0b9b62d..a8912cd4184 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -840,9 +840,11 @@ regen-ast: $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ $(srcdir)/Parser/Python.asdl \ -H $(srcdir)/Include/Python-ast.h.new \ + -I $(srcdir)/Include/internal/pycore_ast.h.new \ -C $(srcdir)/Python/Python-ast.c.new $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new .PHONY: regen-opcode diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst new file mode 100644 index 00000000000..b3ac08edd84 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst @@ -0,0 +1,2 @@ +The :mod:`ast` module internal state is now per interpreter. Patch by Victor +Stinner. diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj index 90d6dc68d54..936f5fd2464 100644 --- a/PCbuild/regen.vcxproj +++ b/PCbuild/regen.vcxproj @@ -137,6 +137,8 @@ + + @@ -161,15 +163,18 @@ - - + + - + + + + @@ -204,6 +209,7 @@ + diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 481261cd853..9a833e841de 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -3,6 +3,7 @@ import os import sys +import textwrap from argparse import ArgumentParser from pathlib import Path @@ -11,7 +12,7 @@ import asdl TABSIZE = 4 MAX_COL = 80 -AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n" +AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" def get_c_type(name): """Return a string for the C name of the type. @@ -414,7 +415,7 @@ class PickleVisitor(EmitVisitor): class Obj2ModPrototypeVisitor(PickleVisitor): def visitProduct(self, prod, name): - code = "static int obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena);" + code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);" self.emit(code % (name, get_c_type(name)), 0) visitSum = visitProduct @@ -424,7 +425,7 @@ class Obj2ModVisitor(PickleVisitor): def funcHeader(self, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("int isinstance;", 1) self.emit("", 0) @@ -506,7 +507,7 @@ class Obj2ModVisitor(PickleVisitor): def visitProduct(self, prod, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: @@ -640,7 +641,7 @@ class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): self.emit_type("%s_type" % name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) if prod.attributes: for a in prod.attributes: self.emit_identifier(a.name) @@ -670,7 +671,7 @@ class PyTypesDeclareVisitor(PickleVisitor): ptype = get_c_type(name) for t in sum.types: self.emit_singleton("%s_singleton" % t.name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -725,7 +726,7 @@ ast_clear(AST_object *self) static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -801,7 +802,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -856,7 +857,7 @@ static PyType_Spec AST_type_spec = { }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -882,7 +883,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base, } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -903,7 +904,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -921,7 +922,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (* return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -932,14 +933,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -954,7 +955,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -965,7 +966,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -974,7 +975,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -983,7 +984,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -998,7 +999,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1014,7 +1015,7 @@ static int add_ast_fields(astmodulestate *state) """, 0, reflow=False) - self.emit("static int init_types(astmodulestate *state)",0) + self.emit("static int init_types(struct ast_state *state)",0) self.emit("{", 0) self.emit("if (state->initialized) return 1;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1) @@ -1093,12 +1094,10 @@ class ASTModuleVisitor(PickleVisitor): self.emit("static int", 0) self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) - self.emit('astmodulestate *state = get_ast_state(m);', 1) - self.emit("", 0) - - self.emit("if (!init_types(state)) {", 1) - self.emit("return -1;", 2) - self.emit("}", 1) + self.emit('struct ast_state *state = get_ast_state();', 1) + self.emit('if (state == NULL) {', 1) + self.emit('return -1;', 2) + self.emit('}', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('return -1;', 2) self.emit('}', 1) @@ -1126,7 +1125,7 @@ static PyModuleDef_Slot astmodule_slots[] = { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -1169,7 +1168,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(astmodulestate *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1206,7 +1205,7 @@ class ObjVisitor(PickleVisitor): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(astmodulestate *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1280,7 +1279,7 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1297,7 +1296,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -1323,7 +1326,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -1341,7 +1344,35 @@ class ChainOfVisitors: v.emit("", 0) -def generate_module_def(f, mod): +def generate_ast_state(module_state, f): + f.write('struct ast_state {\n') + f.write(' int initialized;\n') + for s in module_state: + f.write(' PyObject *' + s + ';\n') + f.write('};') + + +def generate_ast_fini(module_state, f): + f.write(""" +void _PyAST_Fini(PyThreadState *tstate) +{ +#ifdef Py_BUILD_CORE + struct ast_state *state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + +""") + for s in module_state: + f.write(" Py_CLEAR(state->" + s + ');\n') + f.write(""" + state->initialized = 0; +} + +""") + + +def generate_module_def(mod, f, internal_h): # Gather all the data needed for ModuleSpec visitor_list = set() with open(os.devnull, "w") as devnull: @@ -1371,50 +1402,64 @@ def generate_module_def(f, mod): module_state.add(tp) state_strings = sorted(state_strings) module_state = sorted(module_state) - f.write('typedef struct {\n') - f.write(' int initialized;\n') - for s in module_state: - f.write(' PyObject *' + s + ';\n') - f.write('} astmodulestate;\n\n') + + generate_ast_state(module_state, internal_h) + + print(textwrap.dedent(f""" + #ifdef Py_BUILD_CORE + # include "pycore_ast.h" // struct ast_state + # include "pycore_interp.h" // _PyInterpreterState.ast + # include "pycore_pystate.h" // _PyInterpreterState_GET() + #else + """).strip(), file=f) + + generate_ast_state(module_state, f) + + print(textwrap.dedent(f""" + #endif // Py_BUILD_CORE + """).rstrip(), file=f) + f.write(""" // Forward declaration -static int init_types(astmodulestate *state); +static int init_types(struct ast_state *state); -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; - -static astmodulestate* -get_global_ast_state(void) +#ifdef Py_BUILD_CORE +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } +#else +static struct ast_state global_ast_state; -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } return state; } - -void _PyAST_Fini(PyThreadState *tstate) -{ - astmodulestate* state = &global_ast_state; +#endif // Py_BUILD_CORE """) - for s in module_state: - f.write(" Py_CLEAR(state->" + s + ');\n') - f.write(""" - state->initialized = 0; -} + # f-string for {mod.name} + f.write(f""" +// Include {mod.name}-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "{mod.name}-ast.h" +#include "structmember.h" """) - f.write('static int init_identifiers(astmodulestate *state)\n') + + generate_ast_fini(module_state, f) + + f.write('static int init_identifiers(struct ast_state *state)\n') f.write('{\n') for identifier in state_strings: f.write(' if ((state->' + identifier) @@ -1423,7 +1468,7 @@ void _PyAST_Fini(PyThreadState *tstate) f.write(' return 1;\n') f.write('};\n\n') -def write_header(f, mod): +def write_header(mod, f): f.write('#ifndef Py_PYTHON_AST_H\n') f.write('#define Py_PYTHON_AST_H\n') f.write('#ifdef __cplusplus\n') @@ -1452,15 +1497,39 @@ def write_header(f, mod): f.write('#endif\n') f.write('#endif /* !Py_PYTHON_AST_H */\n') -def write_source(f, mod): - f.write('#include \n') - f.write('\n') - f.write('#include "Python.h"\n') - f.write('#include "%s-ast.h"\n' % mod.name) - f.write('#include "structmember.h" // PyMemberDef\n') - f.write('\n') - generate_module_def(f, mod) +def write_internal_h_header(mod, f): + print(textwrap.dedent(""" + #ifndef Py_INTERNAL_AST_H + #define Py_INTERNAL_AST_H + #ifdef __cplusplus + extern "C" { + #endif + + #ifndef Py_BUILD_CORE + # error "this header requires Py_BUILD_CORE define" + #endif + """).lstrip(), file=f) + + +def write_internal_h_footer(mod, f): + print(textwrap.dedent(""" + + #ifdef __cplusplus + } + #endif + #endif /* !Py_INTERNAL_AST_H */ + """), file=f) + + +def write_source(mod, f, internal_h_file): + print(textwrap.dedent(f""" + #include + + #include "Python.h" + """), file=f) + + generate_module_def(mod, f, internal_h_file) v = ChainOfVisitors( SequenceConstructorVisitor(f), @@ -1475,27 +1544,37 @@ def write_source(f, mod): ) v.visit(mod) -def main(input_file, c_file, h_file, dump_module=False): +def main(input_filename, c_filename, h_filename, internal_h_filename, dump_module=False): auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) - mod = asdl.parse(input_file) + mod = asdl.parse(input_filename) if dump_module: print('Parsed Module:') print(mod) if not asdl.check(mod): sys.exit(1) - for file, writer in (c_file, write_source), (h_file, write_header): - if file is not None: - with file.open("w") as f: - f.write(auto_gen_msg) - writer(f, mod) - print(file, "regenerated.") + + with c_filename.open("w") as c_file, \ + h_filename.open("w") as h_file, \ + internal_h_filename.open("w") as internal_h_file: + c_file.write(auto_gen_msg) + h_file.write(auto_gen_msg) + internal_h_file.write(auto_gen_msg) + + write_internal_h_header(mod, internal_h_file) + write_source(mod, c_file, internal_h_file) + write_header(mod, h_file) + write_internal_h_footer(mod, internal_h_file) + + print(f"{c_filename}, {h_filename}, {internal_h_filename} regenerated.") if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) - parser.add_argument("-C", "--c-file", type=Path, default=None) - parser.add_argument("-H", "--h-file", type=Path, default=None) + parser.add_argument("-C", "--c-file", type=Path, required=True) + parser.add_argument("-H", "--h-file", type=Path, required=True) + parser.add_argument("-I", "--internal-h-file", type=Path, required=True) parser.add_argument("-d", "--dump-module", action="store_true") - options = parser.parse_args() - main(**vars(options)) + args = parser.parse_args() + main(args.input_file, args.c_file, args.h_file, + args.internal_h_file, args.dump_module) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 13657a67275..f04addbe201 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1,12 +1,16 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. + #include #include "Python.h" -#include "Python-ast.h" -#include "structmember.h" // PyMemberDef -typedef struct { +#ifdef Py_BUILD_CORE +# include "pycore_ast.h" // struct ast_state +# include "pycore_interp.h" // _PyInterpreterState.ast +# include "pycore_pystate.h" // _PyInterpreterState_GET() +#else +struct ast_state { int initialized; PyObject *AST_type; PyObject *Add_singleton; @@ -221,38 +225,50 @@ typedef struct { PyObject *values; PyObject *vararg; PyObject *withitem_type; -} astmodulestate; - +}; +#endif // Py_BUILD_CORE // Forward declaration -static int init_types(astmodulestate *state); +static int init_types(struct ast_state *state); -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; - -static astmodulestate* -get_global_ast_state(void) +#ifdef Py_BUILD_CORE +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } +#else +static struct ast_state global_ast_state; -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } return state; } +#endif // Py_BUILD_CORE + +// Include Python-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "Python-ast.h" +#include "structmember.h" void _PyAST_Fini(PyThreadState *tstate) { - astmodulestate* state = &global_ast_state; +#ifdef Py_BUILD_CORE + struct ast_state *state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + Py_CLEAR(state->AST_type); Py_CLEAR(state->Add_singleton); Py_CLEAR(state->Add_type); @@ -470,7 +486,7 @@ void _PyAST_Fini(PyThreadState *tstate) state->initialized = 0; } -static int init_identifiers(astmodulestate *state) +static int init_identifiers(struct ast_state *state) { if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; @@ -561,7 +577,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(alias, alias_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) -static PyObject* ast2obj_mod(astmodulestate *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -582,7 +598,7 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(astmodulestate *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -699,7 +715,7 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(astmodulestate *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -812,12 +828,12 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty); -static PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty); -static PyObject* ast2obj_operator(astmodulestate *state, operator_ty); -static PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty); -static PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty); -static PyObject* ast2obj_comprehension(astmodulestate *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -830,13 +846,13 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(astmodulestate *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(astmodulestate *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -846,7 +862,7 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(astmodulestate *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -858,7 +874,7 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(astmodulestate *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -869,17 +885,17 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(astmodulestate *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, void*); static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(astmodulestate *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_type_ignore(astmodulestate *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -923,7 +939,7 @@ ast_clear(AST_object *self) static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -999,7 +1015,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1054,7 +1070,7 @@ static PyType_Spec AST_type_spec = { }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -1080,7 +1096,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base, } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -1101,7 +1117,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1119,7 +1135,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (* return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -1130,14 +1146,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -1152,7 +1168,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -1163,7 +1179,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -1172,7 +1188,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -1181,7 +1197,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -1196,7 +1212,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1211,7 +1227,7 @@ static int add_ast_fields(astmodulestate *state) } -static int init_types(astmodulestate *state) +static int init_types(struct ast_state *state) { if (state->initialized) return 1; if (init_identifiers(state) < 0) return 0; @@ -1890,37 +1906,37 @@ static int init_types(astmodulestate *state) return 1; } -static int obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, +static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena); -static int obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, +static int obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* arena); -static int obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, +static int obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* arena); -static int obj2ast_expr_context(astmodulestate *state, PyObject* obj, +static int obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena); -static int obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, - PyArena* arena); -static int obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* - out, PyArena* arena); -static int obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* +static int obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* + out, PyArena* arena); +static int obj2ast_operator(struct ast_state *state, PyObject* obj, + operator_ty* out, PyArena* arena); +static int obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, PyArena* arena); -static int obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, +static int obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena); -static int obj2ast_comprehension(astmodulestate *state, PyObject* obj, +static int obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena); -static int obj2ast_excepthandler(astmodulestate *state, PyObject* obj, +static int obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena); -static int obj2ast_arguments(astmodulestate *state, PyObject* obj, +static int obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena); -static int obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, +static int obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena); -static int obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* +static int obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, PyArena* arena); -static int obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, +static int obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena); -static int obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* - out, PyArena* arena); -static int obj2ast_type_ignore(astmodulestate *state, PyObject* obj, +static int obj2ast_withitem(struct ast_state *state, PyObject* obj, + withitem_ty* out, PyArena* arena); +static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); mod_ty @@ -3391,7 +3407,7 @@ TypeIgnore(int lineno, string tag, PyArena *arena) PyObject* -ast2obj_mod(astmodulestate *state, void* _o) +ast2obj_mod(struct ast_state *state, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3462,7 +3478,7 @@ failed: } PyObject* -ast2obj_stmt(astmodulestate *state, void* _o) +ast2obj_stmt(struct ast_state *state, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3977,7 +3993,7 @@ failed: } PyObject* -ast2obj_expr(astmodulestate *state, void* _o) +ast2obj_expr(struct ast_state *state, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4453,7 +4469,7 @@ failed: return NULL; } -PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) { switch(o) { case Load: @@ -4468,7 +4484,7 @@ PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) { switch(o) { case And: @@ -4480,7 +4496,7 @@ PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) { switch(o) { case Add: @@ -4525,7 +4541,7 @@ PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) { switch(o) { case Invert: @@ -4543,7 +4559,7 @@ PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) { switch(o) { case Eq: @@ -4580,7 +4596,7 @@ PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(astmodulestate *state, void* _o) +ast2obj_comprehension(struct ast_state *state, void* _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4619,7 +4635,7 @@ failed: } PyObject* -ast2obj_excepthandler(astmodulestate *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, void* _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4678,7 +4694,7 @@ failed: } PyObject* -ast2obj_arguments(astmodulestate *state, void* _o) +ast2obj_arguments(struct ast_state *state, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4732,7 +4748,7 @@ failed: } PyObject* -ast2obj_arg(astmodulestate *state, void* _o) +ast2obj_arg(struct ast_state *state, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4786,7 +4802,7 @@ failed: } PyObject* -ast2obj_keyword(astmodulestate *state, void* _o) +ast2obj_keyword(struct ast_state *state, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4835,7 +4851,7 @@ failed: } PyObject* -ast2obj_alias(astmodulestate *state, void* _o) +ast2obj_alias(struct ast_state *state, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4864,7 +4880,7 @@ failed: } PyObject* -ast2obj_withitem(astmodulestate *state, void* _o) +ast2obj_withitem(struct ast_state *state, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4893,7 +4909,7 @@ failed: } PyObject* -ast2obj_type_ignore(astmodulestate *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4927,7 +4943,7 @@ failed: int -obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) +obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) { int isinstance; @@ -5154,7 +5170,8 @@ obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) } int -obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) +obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* + arena) { int isinstance; @@ -7052,7 +7069,8 @@ obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) } int -obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) +obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* + arena) { int isinstance; @@ -8562,7 +8580,7 @@ obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) } int -obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* +obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena) { int isinstance; @@ -8597,7 +8615,7 @@ obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* } int -obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* +obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* arena) { int isinstance; @@ -8624,7 +8642,7 @@ obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* } int -obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, +obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, PyArena* arena) { int isinstance; @@ -8739,8 +8757,8 @@ obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, } int -obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* - arena) +obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, + PyArena* arena) { int isinstance; @@ -8782,7 +8800,7 @@ obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* } int -obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* +obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena) { int isinstance; @@ -8873,7 +8891,7 @@ obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* } int -obj2ast_comprehension(astmodulestate *state, PyObject* obj, comprehension_ty* +obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -8962,7 +8980,7 @@ failed: } int -obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* +obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena) { int isinstance; @@ -9112,7 +9130,7 @@ obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* } int -obj2ast_arguments(astmodulestate *state, PyObject* obj, arguments_ty* out, +obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9324,7 +9342,7 @@ failed: } int -obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, PyArena* arena) +obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9435,8 +9453,8 @@ failed: } int -obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* out, PyArena* - arena) +obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, + PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9533,7 +9551,7 @@ failed: } int -obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, PyArena* +obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9574,7 +9592,7 @@ failed: } int -obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* out, +obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9615,8 +9633,8 @@ failed: } int -obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, - PyArena* arena) +obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* + out, PyArena* arena) { int isinstance; @@ -9677,9 +9695,8 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, static int astmodule_exec(PyObject *m) { - astmodulestate *state = get_ast_state(m); - - if (!init_types(state)) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { return -1; } if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { @@ -10132,7 +10149,7 @@ static PyModuleDef_Slot astmodule_slots[] = { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -10146,7 +10163,7 @@ PyInit__ast(void) PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -10163,7 +10180,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -10189,7 +10210,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; }