Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-41796: Make _ast module state per interpreter (GH-23024) 

The ast module internal state is now per interpreter.

* Rename "astmodulestate" to "struct ast_state"
* Add pycore_ast.h internal header: the ast_state structure is now
  declared in pycore_ast.h.
* Add PyInterpreterState.ast (struct ast_state)
* Remove get_ast_state()
* Rename get_global_ast_state() to get_ast_state()
* PyAST_obj2mod() now handles get_ast_state() failures
This commit is contained in:
Victor Stinner 2020-11-02 22:03:28 +01:00 committed by GitHub
parent 34637a0ce2
commit 5cf4782a26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 549 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Include/Python-ast.h generated
View File

@ -1,4 +1,4 @@
/* File automatically generated by Parser/asdl_c.py. */ // File automatically generated by Parser/asdl_c.py.
#ifndef Py_PYTHON_AST_H #ifndef Py_PYTHON_AST_H
#define Py_PYTHON_AST_H #define Py_PYTHON_AST_H

234
Include/internal/pycore_ast.h Normal file
View File

@ -0,0 +1,234 @@
// File automatically generated by Parser/asdl_c.py.
#ifndef Py_INTERNAL_AST_H
#define Py_INTERNAL_AST_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
struct ast_state {
int initialized;
PyObject *AST_type;
PyObject *Add_singleton;
PyObject *Add_type;
PyObject *And_singleton;
PyObject *And_type;
PyObject *AnnAssign_type;
PyObject *Assert_type;
PyObject *Assign_type;
PyObject *AsyncFor_type;
PyObject *AsyncFunctionDef_type;
PyObject *AsyncWith_type;
PyObject *Attribute_type;
PyObject *AugAssign_type;
PyObject *Await_type;
PyObject *BinOp_type;
PyObject *BitAnd_singleton;
PyObject *BitAnd_type;
PyObject *BitOr_singleton;
PyObject *BitOr_type;
PyObject *BitXor_singleton;
PyObject *BitXor_type;
PyObject *BoolOp_type;
PyObject *Break_type;
PyObject *Call_type;
PyObject *ClassDef_type;
PyObject *Compare_type;
PyObject *Constant_type;
PyObject *Continue_type;
PyObject *Del_singleton;
PyObject *Del_type;
PyObject *Delete_type;
PyObject *DictComp_type;
PyObject *Dict_type;
PyObject *Div_singleton;
PyObject *Div_type;
PyObject *Eq_singleton;
PyObject *Eq_type;
PyObject *ExceptHandler_type;
PyObject *Expr_type;
PyObject *Expression_type;
PyObject *FloorDiv_singleton;
PyObject *FloorDiv_type;
PyObject *For_type;
PyObject *FormattedValue_type;
PyObject *FunctionDef_type;
PyObject *FunctionType_type;
PyObject *GeneratorExp_type;
PyObject *Global_type;
PyObject *GtE_singleton;
PyObject *GtE_type;
PyObject *Gt_singleton;
PyObject *Gt_type;
PyObject *IfExp_type;
PyObject *If_type;
PyObject *ImportFrom_type;
PyObject *Import_type;
PyObject *In_singleton;
PyObject *In_type;
PyObject *Interactive_type;
PyObject *Invert_singleton;
PyObject *Invert_type;
PyObject *IsNot_singleton;
PyObject *IsNot_type;
PyObject *Is_singleton;
PyObject *Is_type;
PyObject *JoinedStr_type;
PyObject *LShift_singleton;
PyObject *LShift_type;
PyObject *Lambda_type;
PyObject *ListComp_type;
PyObject *List_type;
PyObject *Load_singleton;
PyObject *Load_type;
PyObject *LtE_singleton;
PyObject *LtE_type;
PyObject *Lt_singleton;
PyObject *Lt_type;
PyObject *MatMult_singleton;
PyObject *MatMult_type;
PyObject *Mod_singleton;
PyObject *Mod_type;
PyObject *Module_type;
PyObject *Mult_singleton;
PyObject *Mult_type;
PyObject *Name_type;
PyObject *NamedExpr_type;
PyObject *Nonlocal_type;
PyObject *NotEq_singleton;
PyObject *NotEq_type;
PyObject *NotIn_singleton;
PyObject *NotIn_type;
PyObject *Not_singleton;
PyObject *Not_type;
PyObject *Or_singleton;
PyObject *Or_type;
PyObject *Pass_type;
PyObject *Pow_singleton;
PyObject *Pow_type;
PyObject *RShift_singleton;
PyObject *RShift_type;
PyObject *Raise_type;
PyObject *Return_type;
PyObject *SetComp_type;
PyObject *Set_type;
PyObject *Slice_type;
PyObject *Starred_type;
PyObject *Store_singleton;
PyObject *Store_type;
PyObject *Sub_singleton;
PyObject *Sub_type;
PyObject *Subscript_type;
PyObject *Try_type;
PyObject *Tuple_type;
PyObject *TypeIgnore_type;
PyObject *UAdd_singleton;
PyObject *UAdd_type;
PyObject *USub_singleton;
PyObject *USub_type;
PyObject *UnaryOp_type;
PyObject *While_type;
PyObject *With_type;
PyObject *YieldFrom_type;
PyObject *Yield_type;
PyObject *__dict__;
PyObject *__doc__;
PyObject *__module__;
PyObject *_attributes;
PyObject *_fields;
PyObject *alias_type;
PyObject *annotation;
PyObject *arg;
PyObject *arg_type;
PyObject *args;
PyObject *argtypes;
PyObject *arguments_type;
PyObject *asname;
PyObject *ast;
PyObject *attr;
PyObject *bases;
PyObject *body;
PyObject *boolop_type;
PyObject *cause;
PyObject *cmpop_type;
PyObject *col_offset;
PyObject *comparators;
PyObject *comprehension_type;
PyObject *context_expr;
PyObject *conversion;
PyObject *ctx;
PyObject *decorator_list;
PyObject *defaults;
PyObject *elt;
PyObject *elts;
PyObject *end_col_offset;
PyObject *end_lineno;
PyObject *exc;
PyObject *excepthandler_type;
PyObject *expr_context_type;
PyObject *expr_type;
PyObject *finalbody;
PyObject *format_spec;
PyObject *func;
PyObject *generators;
PyObject *handlers;
PyObject *id;
PyObject *ifs;
PyObject *is_async;
PyObject *items;
PyObject *iter;
PyObject *key;
PyObject *keys;
PyObject *keyword_type;
PyObject *keywords;
PyObject *kind;
PyObject *kw_defaults;
PyObject *kwarg;
PyObject *kwonlyargs;
PyObject *left;
PyObject *level;
PyObject *lineno;
PyObject *lower;
PyObject *mod_type;
PyObject *module;
PyObject *msg;
PyObject *name;
PyObject *names;
PyObject *op;
PyObject *operand;
PyObject *operator_type;
PyObject *ops;
PyObject *optional_vars;
PyObject *orelse;
PyObject *posonlyargs;
PyObject *returns;
PyObject *right;
PyObject *simple;
PyObject *slice;
PyObject *step;
PyObject *stmt_type;
PyObject *tag;
PyObject *target;
PyObject *targets;
PyObject *test;
PyObject *type;
PyObject *type_comment;
PyObject *type_ignore_type;
PyObject *type_ignores;
PyObject *unaryop_type;
PyObject *upper;
PyObject *value;
PyObject *values;
PyObject *vararg;
PyObject *withitem_type;
};
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_AST_H */

11
Include/internal/pycore_interp.h
View File

@ -8,10 +8,11 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define" # error "this header requires Py_BUILD_CORE define"
#endif #endif
#include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_atomic.h" // _Py_atomic_address
#include "pycore_gil.h" /* struct _gil_runtime_state */ #include "pycore_ast.h" // struct ast_state
#include "pycore_gc.h" /* struct _gc_runtime_state */ #include "pycore_gil.h" // struct _gil_runtime_state
#include "pycore_warnings.h" /* struct _warnings_runtime_state */ #include "pycore_gc.h" // struct _gc_runtime_state
#include "pycore_warnings.h" // struct _warnings_runtime_state
struct _pending_calls { struct _pending_calls {
PyThread_type_lock lock; PyThread_type_lock lock;
@ -258,6 +259,8 @@ struct _is {
struct _Py_async_gen_state async_gen; struct _Py_async_gen_state async_gen;
struct _Py_context_state context; struct _Py_context_state context;
struct _Py_exc_state exc_state; struct _Py_exc_state exc_state;
struct ast_state ast;
}; };
/* Used by _PyImport_Cleanup() */ /* Used by _PyImport_Cleanup() */

2
Makefile.pre.in
View File

@ -840,9 +840,11 @@ regen-ast:
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
$(srcdir)/Parser/Python.asdl \ $(srcdir)/Parser/Python.asdl \
-H $(srcdir)/Include/Python-ast.h.new \ -H $(srcdir)/Include/Python-ast.h.new \
-I $(srcdir)/Include/internal/pycore_ast.h.new \
-C $(srcdir)/Python/Python-ast.c.new -C $(srcdir)/Python/Python-ast.c.new
$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
.PHONY: regen-opcode .PHONY: regen-opcode

2
Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst Normal file
View File

@ -0,0 +1,2 @@
The :mod:`ast` module internal state is now per interpreter. Patch by Victor
Stinner.

12
PCbuild/regen.vcxproj
View File

@ -137,6 +137,8 @@
</None> </None>
<None Include="..\Include\opcode.h"> <None Include="..\Include\opcode.h">
</None> </None>
<None Include="..\Include\internal\pycore_ast.h">
</None>
<None Include="..\Include\Python-ast.h"> <None Include="..\Include\Python-ast.h">
</None> </None>
<None Include="..\Python\Python-ast.c"> <None Include="..\Python\Python-ast.c">
@ -161,15 +163,18 @@
<Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" /> <Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
</Target> </Target>
<Target Name="_RegenAST_H" AfterTargets="_RegenGrammar"> <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
<!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h --> <!-- Regenerate Include/Python-ast.h, Python/Python-ast.c and Include/internal/pycore_ast.h using Parser/asdl_c.py -h -->
<Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" /> <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot; -I &quot;$(IntDir)pycore_ast.h&quot;" />
<Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h"> <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" /> <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
</Copy> </Copy>
<Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c"> <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" /> <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
</Copy> </Copy>
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" /> <Copy SourceFiles="$(IntDir)pycore_ast.h" DestinationFiles="$(PySourcePath)Include\internal\pycore_ast.h">
<Output TaskParameter="CopiedFiles" ItemName="_UpdatedInternalH" />
</Copy>
<Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' != '' and '@(_UpdatedC)' != '' and @(_UpdatedInternalH)'" />
</Target> </Target>
<Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C"> <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
<!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py--> <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
@ -204,6 +209,7 @@
<Clean Include="$(IntDir)opcode.h" /> <Clean Include="$(IntDir)opcode.h" />
<Clean Include="$(IntDir)Python-ast.c" /> <Clean Include="$(IntDir)Python-ast.c" />
<Clean Include="$(IntDir)Python-ast.h" /> <Clean Include="$(IntDir)Python-ast.h" />
<Clean Include="$(IntDir)pycore_ast.h" />
</ItemGroup> </ItemGroup>
</Target> </Target>
</Project> </Project>

243
Parser/asdl_c.py
View File

@ -3,6 +3,7 @@
import os import os
import sys import sys
import textwrap
from argparse import ArgumentParser from argparse import ArgumentParser
from pathlib import Path from pathlib import Path
@ -11,7 +12,7 @@ import asdl
TABSIZE = 4 TABSIZE = 4
MAX_COL = 80 MAX_COL = 80
AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n" AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n"
def get_c_type(name): def get_c_type(name):
"""Return a string for the C name of the type. """Return a string for the C name of the type.
@ -414,7 +415,7 @@ class PickleVisitor(EmitVisitor):
class Obj2ModPrototypeVisitor(PickleVisitor): class Obj2ModPrototypeVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
code = "static int obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena);" code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);"
self.emit(code % (name, get_c_type(name)), 0) self.emit(code % (name, get_c_type(name)), 0)
visitSum = visitProduct visitSum = visitProduct
@ -424,7 +425,7 @@ class Obj2ModVisitor(PickleVisitor):
def funcHeader(self, name): def funcHeader(self, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("int", 0) self.emit("int", 0)
self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("int isinstance;", 1) self.emit("int isinstance;", 1)
self.emit("", 0) self.emit("", 0)
@ -506,7 +507,7 @@ class Obj2ModVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("int", 0) self.emit("int", 0)
self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("PyObject* tmp = NULL;", 1) self.emit("PyObject* tmp = NULL;", 1)
for f in prod.fields: for f in prod.fields:
@ -640,7 +641,7 @@ class PyTypesDeclareVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
self.emit_type("%s_type" % name) self.emit_type("%s_type" % name)
self.emit("static PyObject* ast2obj_%s(astmodulestate *state, void*);" % name, 0) self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0)
if prod.attributes: if prod.attributes:
for a in prod.attributes: for a in prod.attributes:
self.emit_identifier(a.name) self.emit_identifier(a.name)
@ -670,7 +671,7 @@ class PyTypesDeclareVisitor(PickleVisitor):
ptype = get_c_type(name) ptype = get_c_type(name)
for t in sum.types: for t in sum.types:
self.emit_singleton("%s_singleton" % t.name) self.emit_singleton("%s_singleton" % t.name)
self.emit("static PyObject* ast2obj_%s(astmodulestate *state, %s);" % (name, ptype), 0) self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0)
for t in sum.types: for t in sum.types:
self.visitConstructor(t, name) self.visitConstructor(t, name)
@ -725,7 +726,7 @@ ast_clear(AST_object *self)
static int static int
ast_type_init(PyObject *self, PyObject *args, PyObject *kw) ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return -1; return -1;
} }
@ -801,7 +802,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
static PyObject * static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused) ast_type_reduce(PyObject *self, PyObject *unused)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return NULL; return NULL;
} }
@ -856,7 +857,7 @@ static PyType_Spec AST_type_spec = {
}; };
static PyObject * static PyObject *
make_type(astmodulestate *state, const char *type, PyObject* base, make_type(struct ast_state *state, const char *type, PyObject* base,
const char* const* fields, int num_fields, const char *doc) const char* const* fields, int num_fields, const char *doc)
{ {
PyObject *fnames, *result; PyObject *fnames, *result;
@ -882,7 +883,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base,
} }
static int static int
add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields)
{ {
int i, result; int i, result;
PyObject *s, *l = PyTuple_New(num_fields); PyObject *s, *l = PyTuple_New(num_fields);
@ -903,7 +904,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs,
/* Conversion AST -> Python */ /* Conversion AST -> Python */
static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*))
{ {
Py_ssize_t i, n = asdl_seq_LEN(seq); Py_ssize_t i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n); PyObject *result = PyList_New(n);
@ -921,7 +922,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*
return result; return result;
} }
static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o)
{ {
if (!o) if (!o)
o = Py_None; o = Py_None;
@ -932,14 +933,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o)
#define ast2obj_identifier ast2obj_object #define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object #define ast2obj_string ast2obj_object
static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b)
{ {
return PyLong_FromLong(b); return PyLong_FromLong(b);
} }
/* Conversion Python -> AST */ /* Conversion Python -> AST */
static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (obj == Py_None) if (obj == Py_None)
obj = NULL; obj = NULL;
@ -954,7 +955,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj
return 0; return 0;
} }
static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (PyArena_AddPyObject(arena, obj) < 0) { if (PyArena_AddPyObject(arena, obj) < 0) {
*out = NULL; *out = NULL;
@ -965,7 +966,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO
return 0; return 0;
} }
static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && obj != Py_None) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str");
@ -974,7 +975,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o
return obj2ast_object(state, obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
@ -983,7 +984,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out,
return obj2ast_object(state, obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena)
{ {
int i; int i;
if (!PyLong_Check(obj)) { if (!PyLong_Check(obj)) {
@ -998,7 +999,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out
return 0; return 0;
} }
static int add_ast_fields(astmodulestate *state) static int add_ast_fields(struct ast_state *state)
{ {
PyObject *empty_tuple; PyObject *empty_tuple;
empty_tuple = PyTuple_New(0); empty_tuple = PyTuple_New(0);
@ -1014,7 +1015,7 @@ static int add_ast_fields(astmodulestate *state)
""", 0, reflow=False) """, 0, reflow=False)
self.emit("static int init_types(astmodulestate *state)",0) self.emit("static int init_types(struct ast_state *state)",0)
self.emit("{", 0) self.emit("{", 0)
self.emit("if (state->initialized) return 1;", 1) self.emit("if (state->initialized) return 1;", 1)
self.emit("if (init_identifiers(state) < 0) return 0;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1)
@ -1093,12 +1094,10 @@ class ASTModuleVisitor(PickleVisitor):
self.emit("static int", 0) self.emit("static int", 0)
self.emit("astmodule_exec(PyObject *m)", 0) self.emit("astmodule_exec(PyObject *m)", 0)
self.emit("{", 0) self.emit("{", 0)
self.emit('astmodulestate *state = get_ast_state(m);', 1) self.emit('struct ast_state *state = get_ast_state();', 1)
self.emit("", 0) self.emit('if (state == NULL) {', 1)
self.emit('return -1;', 2)
self.emit("if (!init_types(state)) {", 1) self.emit('}', 1)
self.emit("return -1;", 2)
self.emit("}", 1)
self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1)
self.emit('return -1;', 2) self.emit('return -1;', 2)
self.emit('}', 1) self.emit('}', 1)
@ -1126,7 +1125,7 @@ static PyModuleDef_Slot astmodule_slots[] = {
static struct PyModuleDef _astmodule = { static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT, PyModuleDef_HEAD_INIT,
.m_name = "_ast", .m_name = "_ast",
// The _ast module uses a global state (global_ast_state). // The _ast module uses a per-interpreter state (PyInterpreterState.ast)
.m_size = 0, .m_size = 0,
.m_slots = astmodule_slots, .m_slots = astmodule_slots,
}; };
@ -1169,7 +1168,7 @@ class ObjVisitor(PickleVisitor):
def func_begin(self, name): def func_begin(self, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("PyObject*", 0) self.emit("PyObject*", 0)
self.emit("ast2obj_%s(astmodulestate *state, void* _o)" % (name), 0) self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
self.emit("PyObject *result = NULL, *value = NULL;", 1) self.emit("PyObject *result = NULL, *value = NULL;", 1)
@ -1206,7 +1205,7 @@ class ObjVisitor(PickleVisitor):
self.func_end() self.func_end()
def simpleSum(self, sum, name): def simpleSum(self, sum, name):
self.emit("PyObject* ast2obj_%s(astmodulestate *state, %s_ty o)" % (name, name), 0) self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("switch(o) {", 1) self.emit("switch(o) {", 1)
for t in sum.types: for t in sum.types:
@ -1280,7 +1279,7 @@ class PartingShots(StaticVisitor):
CODE = """ CODE = """
PyObject* PyAST_mod2obj(mod_ty t) PyObject* PyAST_mod2obj(mod_ty t)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return NULL; return NULL;
} }
@ -1297,7 +1296,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return NULL; return NULL;
} }
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) {
return NULL;
}
PyObject *req_type[3]; PyObject *req_type[3];
req_type[0] = state->Module_type; req_type[0] = state->Module_type;
req_type[1] = state->Expression_type; req_type[1] = state->Expression_type;
@ -1323,7 +1326,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
int PyAST_Check(PyObject* obj) int PyAST_Check(PyObject* obj)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return -1; return -1;
} }
@ -1341,7 +1344,35 @@ class ChainOfVisitors:
v.emit("", 0) v.emit("", 0)
def generate_module_def(f, mod): def generate_ast_state(module_state, f):
f.write('struct ast_state {\n')
f.write(' int initialized;\n')
for s in module_state:
f.write(' PyObject *' + s + ';\n')
f.write('};')
def generate_ast_fini(module_state, f):
f.write("""
void _PyAST_Fini(PyThreadState *tstate)
{
#ifdef Py_BUILD_CORE
struct ast_state *state = &tstate->interp->ast;
#else
struct ast_state *state = &global_ast_state;
#endif
""")
for s in module_state:
f.write(" Py_CLEAR(state->" + s + ');\n')
f.write("""
state->initialized = 0;
}
""")
def generate_module_def(mod, f, internal_h):
# Gather all the data needed for ModuleSpec # Gather all the data needed for ModuleSpec
visitor_list = set() visitor_list = set()
with open(os.devnull, "w") as devnull: with open(os.devnull, "w") as devnull:
@ -1371,50 +1402,64 @@ def generate_module_def(f, mod):
module_state.add(tp) module_state.add(tp)
state_strings = sorted(state_strings) state_strings = sorted(state_strings)
module_state = sorted(module_state) module_state = sorted(module_state)
f.write('typedef struct {\n')
f.write(' int initialized;\n') generate_ast_state(module_state, internal_h)
for s in module_state:
f.write(' PyObject *' + s + ';\n') print(textwrap.dedent(f"""
f.write('} astmodulestate;\n\n') #ifdef Py_BUILD_CORE
# include "pycore_ast.h" // struct ast_state
# include "pycore_interp.h" // _PyInterpreterState.ast
# include "pycore_pystate.h" // _PyInterpreterState_GET()
#else
""").strip(), file=f)
generate_ast_state(module_state, f)
print(textwrap.dedent(f"""
#endif // Py_BUILD_CORE
""").rstrip(), file=f)
f.write(""" f.write("""
// Forward declaration // Forward declaration
static int init_types(astmodulestate *state); static int init_types(struct ast_state *state);
// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. #ifdef Py_BUILD_CORE
static astmodulestate global_ast_state = {0}; static struct ast_state*
get_ast_state(void)
static astmodulestate*
get_global_ast_state(void)
{ {
astmodulestate* state = &global_ast_state; PyInterpreterState *interp = _PyInterpreterState_GET();
struct ast_state *state = &interp->ast;
if (!init_types(state)) { if (!init_types(state)) {
return NULL; return NULL;
} }
return state; return state;
} }
#else
static struct ast_state global_ast_state;
static astmodulestate* static struct ast_state*
get_ast_state(PyObject* Py_UNUSED(module)) get_ast_state(void)
{ {
astmodulestate* state = get_global_ast_state(); struct ast_state *state = &global_ast_state;
// get_ast_state() must only be called after _ast module is imported, if (!init_types(state)) {
// and astmodule_exec() calls init_types() return NULL;
assert(state != NULL); }
return state; return state;
} }
#endif // Py_BUILD_CORE
void _PyAST_Fini(PyThreadState *tstate)
{
astmodulestate* state = &global_ast_state;
""") """)
for s in module_state:
f.write(" Py_CLEAR(state->" + s + ');\n')
f.write("""
state->initialized = 0;
}
# f-string for {mod.name}
f.write(f"""
// Include {mod.name}-ast.h after pycore_interp.h to avoid conflicts
// with the Yield macro redefined by <winbase.h>
#include "{mod.name}-ast.h"
#include "structmember.h"
""") """)
f.write('static int init_identifiers(astmodulestate *state)\n')
generate_ast_fini(module_state, f)
f.write('static int init_identifiers(struct ast_state *state)\n')
f.write('{\n') f.write('{\n')
for identifier in state_strings: for identifier in state_strings:
f.write(' if ((state->' + identifier) f.write(' if ((state->' + identifier)
@ -1423,7 +1468,7 @@ void _PyAST_Fini(PyThreadState *tstate)
f.write(' return 1;\n') f.write(' return 1;\n')
f.write('};\n\n') f.write('};\n\n')
def write_header(f, mod): def write_header(mod, f):
f.write('#ifndef Py_PYTHON_AST_H\n') f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n') f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n') f.write('#ifdef __cplusplus\n')
@ -1452,15 +1497,39 @@ def write_header(f, mod):
f.write('#endif\n') f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n') f.write('#endif /* !Py_PYTHON_AST_H */\n')
def write_source(f, mod):
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h" // PyMemberDef\n')
f.write('\n')
generate_module_def(f, mod) def write_internal_h_header(mod, f):
print(textwrap.dedent("""
#ifndef Py_INTERNAL_AST_H
#define Py_INTERNAL_AST_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
""").lstrip(), file=f)
def write_internal_h_footer(mod, f):
print(textwrap.dedent("""
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_AST_H */
"""), file=f)
def write_source(mod, f, internal_h_file):
print(textwrap.dedent(f"""
#include <stddef.h>
#include "Python.h"
"""), file=f)
generate_module_def(mod, f, internal_h_file)
v = ChainOfVisitors( v = ChainOfVisitors(
SequenceConstructorVisitor(f), SequenceConstructorVisitor(f),
@ -1475,27 +1544,37 @@ def write_source(f, mod):
) )
v.visit(mod) v.visit(mod)
def main(input_file, c_file, h_file, dump_module=False): def main(input_filename, c_filename, h_filename, internal_h_filename, dump_module=False):
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
mod = asdl.parse(input_file) mod = asdl.parse(input_filename)
if dump_module: if dump_module:
print('Parsed Module:') print('Parsed Module:')
print(mod) print(mod)
if not asdl.check(mod): if not asdl.check(mod):
sys.exit(1) sys.exit(1)
for file, writer in (c_file, write_source), (h_file, write_header):
if file is not None: with c_filename.open("w") as c_file, \
with file.open("w") as f: h_filename.open("w") as h_file, \
f.write(auto_gen_msg) internal_h_filename.open("w") as internal_h_file:
writer(f, mod) c_file.write(auto_gen_msg)
print(file, "regenerated.") h_file.write(auto_gen_msg)
internal_h_file.write(auto_gen_msg)
write_internal_h_header(mod, internal_h_file)
write_source(mod, c_file, internal_h_file)
write_header(mod, h_file)
write_internal_h_footer(mod, internal_h_file)
print(f"{c_filename}, {h_filename}, {internal_h_filename} regenerated.")
if __name__ == "__main__": if __name__ == "__main__":
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument("input_file", type=Path) parser.add_argument("input_file", type=Path)
parser.add_argument("-C", "--c-file", type=Path, default=None) parser.add_argument("-C", "--c-file", type=Path, required=True)
parser.add_argument("-H", "--h-file", type=Path, default=None) parser.add_argument("-H", "--h-file", type=Path, required=True)
parser.add_argument("-I", "--internal-h-file", type=Path, required=True)
parser.add_argument("-d", "--dump-module", action="store_true") parser.add_argument("-d", "--dump-module", action="store_true")
options = parser.parse_args() args = parser.parse_args()
main(**vars(options)) main(args.input_file, args.c_file, args.h_file,
args.internal_h_file, args.dump_module)

245
Python/Python-ast.c generated
View File

@ -1,12 +1,16 @@
/* File automatically generated by Parser/asdl_c.py. */ // File automatically generated by Parser/asdl_c.py.
#include <stddef.h> #include <stddef.h>
#include "Python.h" #include "Python.h"
#include "Python-ast.h"
#include "structmember.h" // PyMemberDef
typedef struct { #ifdef Py_BUILD_CORE
# include "pycore_ast.h" // struct ast_state
# include "pycore_interp.h" // _PyInterpreterState.ast
# include "pycore_pystate.h" // _PyInterpreterState_GET()
#else
struct ast_state {
int initialized; int initialized;
PyObject *AST_type; PyObject *AST_type;
PyObject *Add_singleton; PyObject *Add_singleton;
@ -221,38 +225,50 @@ typedef struct {
PyObject *values; PyObject *values;
PyObject *vararg; PyObject *vararg;
PyObject *withitem_type; PyObject *withitem_type;
} astmodulestate; };
#endif // Py_BUILD_CORE
// Forward declaration // Forward declaration
static int init_types(astmodulestate *state); static int init_types(struct ast_state *state);
// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. #ifdef Py_BUILD_CORE
static astmodulestate global_ast_state = {0}; static struct ast_state*
get_ast_state(void)
static astmodulestate*
get_global_ast_state(void)
{ {
astmodulestate* state = &global_ast_state; PyInterpreterState *interp = _PyInterpreterState_GET();
struct ast_state *state = &interp->ast;
if (!init_types(state)) { if (!init_types(state)) {
return NULL; return NULL;
} }
return state; return state;
} }
#else
static struct ast_state global_ast_state;
static astmodulestate* static struct ast_state*
get_ast_state(PyObject* Py_UNUSED(module)) get_ast_state(void)
{ {
astmodulestate* state = get_global_ast_state(); struct ast_state *state = &global_ast_state;
// get_ast_state() must only be called after _ast module is imported, if (!init_types(state)) {
// and astmodule_exec() calls init_types() return NULL;
assert(state != NULL); }
return state; return state;
} }
#endif // Py_BUILD_CORE
// Include Python-ast.h after pycore_interp.h to avoid conflicts
// with the Yield macro redefined by <winbase.h>
#include "Python-ast.h"
#include "structmember.h"
void _PyAST_Fini(PyThreadState *tstate) void _PyAST_Fini(PyThreadState *tstate)
{ {
astmodulestate* state = &global_ast_state; #ifdef Py_BUILD_CORE
struct ast_state *state = &tstate->interp->ast;
#else
struct ast_state *state = &global_ast_state;
#endif
Py_CLEAR(state->AST_type); Py_CLEAR(state->AST_type);
Py_CLEAR(state->Add_singleton); Py_CLEAR(state->Add_singleton);
Py_CLEAR(state->Add_type); Py_CLEAR(state->Add_type);
@ -470,7 +486,7 @@ void _PyAST_Fini(PyThreadState *tstate)
state->initialized = 0; state->initialized = 0;
} }
static int init_identifiers(astmodulestate *state) static int init_identifiers(struct ast_state *state)
{ {
if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0;
if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0;
@ -561,7 +577,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(alias, alias_ty)
GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty)
GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty)
static PyObject* ast2obj_mod(astmodulestate *state, void*); static PyObject* ast2obj_mod(struct ast_state *state, void*);
static const char * const Module_fields[]={ static const char * const Module_fields[]={
"body", "body",
"type_ignores", "type_ignores",
@ -582,7 +598,7 @@ static const char * const stmt_attributes[] = {
"end_lineno", "end_lineno",
"end_col_offset", "end_col_offset",
}; };
static PyObject* ast2obj_stmt(astmodulestate *state, void*); static PyObject* ast2obj_stmt(struct ast_state *state, void*);
static const char * const FunctionDef_fields[]={ static const char * const FunctionDef_fields[]={
"name", "name",
"args", "args",
@ -699,7 +715,7 @@ static const char * const expr_attributes[] = {
"end_lineno", "end_lineno",
"end_col_offset", "end_col_offset",
}; };
static PyObject* ast2obj_expr(astmodulestate *state, void*); static PyObject* ast2obj_expr(struct ast_state *state, void*);
static const char * const BoolOp_fields[]={ static const char * const BoolOp_fields[]={
"op", "op",
"values", "values",
@ -812,12 +828,12 @@ static const char * const Slice_fields[]={
"upper", "upper",
"step", "step",
}; };
static PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty); static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty);
static PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty); static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty);
static PyObject* ast2obj_operator(astmodulestate *state, operator_ty); static PyObject* ast2obj_operator(struct ast_state *state, operator_ty);
static PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty); static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty);
static PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty); static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty);
static PyObject* ast2obj_comprehension(astmodulestate *state, void*); static PyObject* ast2obj_comprehension(struct ast_state *state, void*);
static const char * const comprehension_fields[]={ static const char * const comprehension_fields[]={
"target", "target",
"iter", "iter",
@ -830,13 +846,13 @@ static const char * const excepthandler_attributes[] = {
"end_lineno", "end_lineno",
"end_col_offset", "end_col_offset",
}; };
static PyObject* ast2obj_excepthandler(astmodulestate *state, void*); static PyObject* ast2obj_excepthandler(struct ast_state *state, void*);
static const char * const ExceptHandler_fields[]={ static const char * const ExceptHandler_fields[]={
"type", "type",
"name", "name",
"body", "body",
}; };
static PyObject* ast2obj_arguments(astmodulestate *state, void*); static PyObject* ast2obj_arguments(struct ast_state *state, void*);
static const char * const arguments_fields[]={ static const char * const arguments_fields[]={
"posonlyargs", "posonlyargs",
"args", "args",
@ -846,7 +862,7 @@ static const char * const arguments_fields[]={
"kwarg", "kwarg",
"defaults", "defaults",
}; };
static PyObject* ast2obj_arg(astmodulestate *state, void*); static PyObject* ast2obj_arg(struct ast_state *state, void*);
static const char * const arg_attributes[] = { static const char * const arg_attributes[] = {
"lineno", "lineno",
"col_offset", "col_offset",
@ -858,7 +874,7 @@ static const char * const arg_fields[]={
"annotation", "annotation",
"type_comment", "type_comment",
}; };
static PyObject* ast2obj_keyword(astmodulestate *state, void*); static PyObject* ast2obj_keyword(struct ast_state *state, void*);
static const char * const keyword_attributes[] = { static const char * const keyword_attributes[] = {
"lineno", "lineno",
"col_offset", "col_offset",
@ -869,17 +885,17 @@ static const char * const keyword_fields[]={
"arg", "arg",
"value", "value",
}; };
static PyObject* ast2obj_alias(astmodulestate *state, void*); static PyObject* ast2obj_alias(struct ast_state *state, void*);
static const char * const alias_fields[]={ static const char * const alias_fields[]={
"name", "name",
"asname", "asname",
}; };
static PyObject* ast2obj_withitem(astmodulestate *state, void*); static PyObject* ast2obj_withitem(struct ast_state *state, void*);
static const char * const withitem_fields[]={ static const char * const withitem_fields[]={
"context_expr", "context_expr",
"optional_vars", "optional_vars",
}; };
static PyObject* ast2obj_type_ignore(astmodulestate *state, void*); static PyObject* ast2obj_type_ignore(struct ast_state *state, void*);
static const char * const TypeIgnore_fields[]={ static const char * const TypeIgnore_fields[]={
"lineno", "lineno",
"tag", "tag",
@ -923,7 +939,7 @@ ast_clear(AST_object *self)
static int static int
ast_type_init(PyObject *self, PyObject *args, PyObject *kw) ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return -1; return -1;
} }
@ -999,7 +1015,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
static PyObject * static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused) ast_type_reduce(PyObject *self, PyObject *unused)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return NULL; return NULL;
} }
@ -1054,7 +1070,7 @@ static PyType_Spec AST_type_spec = {
}; };
static PyObject * static PyObject *
make_type(astmodulestate *state, const char *type, PyObject* base, make_type(struct ast_state *state, const char *type, PyObject* base,
const char* const* fields, int num_fields, const char *doc) const char* const* fields, int num_fields, const char *doc)
{ {
PyObject *fnames, *result; PyObject *fnames, *result;
@ -1080,7 +1096,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base,
} }
static int static int
add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields)
{ {
int i, result; int i, result;
PyObject *s, *l = PyTuple_New(num_fields); PyObject *s, *l = PyTuple_New(num_fields);
@ -1101,7 +1117,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs,
/* Conversion AST -> Python */ /* Conversion AST -> Python */
static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*))
{ {
Py_ssize_t i, n = asdl_seq_LEN(seq); Py_ssize_t i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n); PyObject *result = PyList_New(n);
@ -1119,7 +1135,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*
return result; return result;
} }
static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o)
{ {
if (!o) if (!o)
o = Py_None; o = Py_None;
@ -1130,14 +1146,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o)
#define ast2obj_identifier ast2obj_object #define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object #define ast2obj_string ast2obj_object
static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b)
{ {
return PyLong_FromLong(b); return PyLong_FromLong(b);
} }
/* Conversion Python -> AST */ /* Conversion Python -> AST */
static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (obj == Py_None) if (obj == Py_None)
obj = NULL; obj = NULL;
@ -1152,7 +1168,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj
return 0; return 0;
} }
static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (PyArena_AddPyObject(arena, obj) < 0) { if (PyArena_AddPyObject(arena, obj) < 0) {
*out = NULL; *out = NULL;
@ -1163,7 +1179,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO
return 0; return 0;
} }
static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && obj != Py_None) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str");
@ -1172,7 +1188,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o
return obj2ast_object(state, obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
@ -1181,7 +1197,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out,
return obj2ast_object(state, obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena)
{ {
int i; int i;
if (!PyLong_Check(obj)) { if (!PyLong_Check(obj)) {
@ -1196,7 +1212,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out
return 0; return 0;
} }
static int add_ast_fields(astmodulestate *state) static int add_ast_fields(struct ast_state *state)
{ {
PyObject *empty_tuple; PyObject *empty_tuple;
empty_tuple = PyTuple_New(0); empty_tuple = PyTuple_New(0);
@ -1211,7 +1227,7 @@ static int add_ast_fields(astmodulestate *state)
} }
static int init_types(astmodulestate *state) static int init_types(struct ast_state *state)
{ {
if (state->initialized) return 1; if (state->initialized) return 1;
if (init_identifiers(state) < 0) return 0; if (init_identifiers(state) < 0) return 0;
@ -1890,37 +1906,37 @@ static int init_types(astmodulestate *state)
return 1; return 1;
} }
static int obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, static int obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, static int obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_expr_context(astmodulestate *state, PyObject* obj, static int obj2ast_expr_context(struct ast_state *state, PyObject* obj,
expr_context_ty* out, PyArena* arena); expr_context_ty* out, PyArena* arena);
static int obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, static int obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty*
PyArena* arena); out, PyArena* arena);
static int obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* static int obj2ast_operator(struct ast_state *state, PyObject* obj,
out, PyArena* arena); operator_ty* out, PyArena* arena);
static int obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* static int obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty*
out, PyArena* arena); out, PyArena* arena);
static int obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, static int obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_comprehension(astmodulestate *state, PyObject* obj, static int obj2ast_comprehension(struct ast_state *state, PyObject* obj,
comprehension_ty* out, PyArena* arena); comprehension_ty* out, PyArena* arena);
static int obj2ast_excepthandler(astmodulestate *state, PyObject* obj, static int obj2ast_excepthandler(struct ast_state *state, PyObject* obj,
excepthandler_ty* out, PyArena* arena); excepthandler_ty* out, PyArena* arena);
static int obj2ast_arguments(astmodulestate *state, PyObject* obj, static int obj2ast_arguments(struct ast_state *state, PyObject* obj,
arguments_ty* out, PyArena* arena); arguments_ty* out, PyArena* arena);
static int obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, static int obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* static int obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty*
out, PyArena* arena); out, PyArena* arena);
static int obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, static int obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out,
PyArena* arena); PyArena* arena);
static int obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* static int obj2ast_withitem(struct ast_state *state, PyObject* obj,
out, PyArena* arena); withitem_ty* out, PyArena* arena);
static int obj2ast_type_ignore(astmodulestate *state, PyObject* obj, static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj,
type_ignore_ty* out, PyArena* arena); type_ignore_ty* out, PyArena* arena);
mod_ty mod_ty
@ -3391,7 +3407,7 @@ TypeIgnore(int lineno, string tag, PyArena *arena)
PyObject* PyObject*
ast2obj_mod(astmodulestate *state, void* _o) ast2obj_mod(struct ast_state *state, void* _o)
{ {
mod_ty o = (mod_ty)_o; mod_ty o = (mod_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -3462,7 +3478,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_stmt(astmodulestate *state, void* _o) ast2obj_stmt(struct ast_state *state, void* _o)
{ {
stmt_ty o = (stmt_ty)_o; stmt_ty o = (stmt_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -3977,7 +3993,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_expr(astmodulestate *state, void* _o) ast2obj_expr(struct ast_state *state, void* _o)
{ {
expr_ty o = (expr_ty)_o; expr_ty o = (expr_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4453,7 +4469,7 @@ failed:
return NULL; return NULL;
} }
PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o)
{ {
switch(o) { switch(o) {
case Load: case Load:
@ -4468,7 +4484,7 @@ PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o)
} }
Py_UNREACHABLE(); Py_UNREACHABLE();
} }
PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o)
{ {
switch(o) { switch(o) {
case And: case And:
@ -4480,7 +4496,7 @@ PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o)
} }
Py_UNREACHABLE(); Py_UNREACHABLE();
} }
PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) PyObject* ast2obj_operator(struct ast_state *state, operator_ty o)
{ {
switch(o) { switch(o) {
case Add: case Add:
@ -4525,7 +4541,7 @@ PyObject* ast2obj_operator(astmodulestate *state, operator_ty o)
} }
Py_UNREACHABLE(); Py_UNREACHABLE();
} }
PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o)
{ {
switch(o) { switch(o) {
case Invert: case Invert:
@ -4543,7 +4559,7 @@ PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o)
} }
Py_UNREACHABLE(); Py_UNREACHABLE();
} }
PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o)
{ {
switch(o) { switch(o) {
case Eq: case Eq:
@ -4580,7 +4596,7 @@ PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o)
Py_UNREACHABLE(); Py_UNREACHABLE();
} }
PyObject* PyObject*
ast2obj_comprehension(astmodulestate *state, void* _o) ast2obj_comprehension(struct ast_state *state, void* _o)
{ {
comprehension_ty o = (comprehension_ty)_o; comprehension_ty o = (comprehension_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4619,7 +4635,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_excepthandler(astmodulestate *state, void* _o) ast2obj_excepthandler(struct ast_state *state, void* _o)
{ {
excepthandler_ty o = (excepthandler_ty)_o; excepthandler_ty o = (excepthandler_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4678,7 +4694,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_arguments(astmodulestate *state, void* _o) ast2obj_arguments(struct ast_state *state, void* _o)
{ {
arguments_ty o = (arguments_ty)_o; arguments_ty o = (arguments_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4732,7 +4748,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_arg(astmodulestate *state, void* _o) ast2obj_arg(struct ast_state *state, void* _o)
{ {
arg_ty o = (arg_ty)_o; arg_ty o = (arg_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4786,7 +4802,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_keyword(astmodulestate *state, void* _o) ast2obj_keyword(struct ast_state *state, void* _o)
{ {
keyword_ty o = (keyword_ty)_o; keyword_ty o = (keyword_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4835,7 +4851,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_alias(astmodulestate *state, void* _o) ast2obj_alias(struct ast_state *state, void* _o)
{ {
alias_ty o = (alias_ty)_o; alias_ty o = (alias_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4864,7 +4880,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_withitem(astmodulestate *state, void* _o) ast2obj_withitem(struct ast_state *state, void* _o)
{ {
withitem_ty o = (withitem_ty)_o; withitem_ty o = (withitem_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4893,7 +4909,7 @@ failed:
} }
PyObject* PyObject*
ast2obj_type_ignore(astmodulestate *state, void* _o) ast2obj_type_ignore(struct ast_state *state, void* _o)
{ {
type_ignore_ty o = (type_ignore_ty)_o; type_ignore_ty o = (type_ignore_ty)_o;
PyObject *result = NULL, *value = NULL; PyObject *result = NULL, *value = NULL;
@ -4927,7 +4943,7 @@ failed:
int int
obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena)
{ {
int isinstance; int isinstance;
@ -5154,7 +5170,8 @@ obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena)
} }
int int
obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena*
arena)
{ {
int isinstance; int isinstance;
@ -7052,7 +7069,8 @@ obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena)
} }
int int
obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena*
arena)
{ {
int isinstance; int isinstance;
@ -8562,7 +8580,7 @@ obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena)
} }
int int
obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty*
out, PyArena* arena) out, PyArena* arena)
{ {
int isinstance; int isinstance;
@ -8597,7 +8615,7 @@ obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty*
} }
int int
obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena*
arena) arena)
{ {
int isinstance; int isinstance;
@ -8624,7 +8642,7 @@ obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena*
} }
int int
obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out,
PyArena* arena) PyArena* arena)
{ {
int isinstance; int isinstance;
@ -8739,8 +8757,8 @@ obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out,
} }
int int
obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out,
arena) PyArena* arena)
{ {
int isinstance; int isinstance;
@ -8782,7 +8800,7 @@ obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena*
} }
int int
obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena*
arena) arena)
{ {
int isinstance; int isinstance;
@ -8873,7 +8891,7 @@ obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena*
} }
int int
obj2ast_comprehension(astmodulestate *state, PyObject* obj, comprehension_ty* obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty*
out, PyArena* arena) out, PyArena* arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
@ -8962,7 +8980,7 @@ failed:
} }
int int
obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty*
out, PyArena* arena) out, PyArena* arena)
{ {
int isinstance; int isinstance;
@ -9112,7 +9130,7 @@ obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty*
} }
int int
obj2ast_arguments(astmodulestate *state, PyObject* obj, arguments_ty* out, obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out,
PyArena* arena) PyArena* arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
@ -9324,7 +9342,7 @@ failed:
} }
int int
obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, PyArena* arena) obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
identifier arg; identifier arg;
@ -9435,8 +9453,8 @@ failed:
} }
int int
obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* out, PyArena* obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out,
arena) PyArena* arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
identifier arg; identifier arg;
@ -9533,7 +9551,7 @@ failed:
} }
int int
obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, PyArena* obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena*
arena) arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
@ -9574,7 +9592,7 @@ failed:
} }
int int
obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* out, obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out,
PyArena* arena) PyArena* arena)
{ {
PyObject* tmp = NULL; PyObject* tmp = NULL;
@ -9615,8 +9633,8 @@ failed:
} }
int int
obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty*
PyArena* arena) out, PyArena* arena)
{ {
int isinstance; int isinstance;
@ -9677,9 +9695,8 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out,
static int static int
astmodule_exec(PyObject *m) astmodule_exec(PyObject *m)
{ {
astmodulestate *state = get_ast_state(m); struct ast_state *state = get_ast_state();
if (state == NULL) {
if (!init_types(state)) {
return -1; return -1;
} }
if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {
@ -10132,7 +10149,7 @@ static PyModuleDef_Slot astmodule_slots[] = {
static struct PyModuleDef _astmodule = { static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT, PyModuleDef_HEAD_INIT,
.m_name = "_ast", .m_name = "_ast",
// The _ast module uses a global state (global_ast_state). // The _ast module uses a per-interpreter state (PyInterpreterState.ast)
.m_size = 0, .m_size = 0,
.m_slots = astmodule_slots, .m_slots = astmodule_slots,
}; };
@ -10146,7 +10163,7 @@ PyInit__ast(void)
PyObject* PyAST_mod2obj(mod_ty t) PyObject* PyAST_mod2obj(mod_ty t)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return NULL; return NULL;
} }
@ -10163,7 +10180,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return NULL; return NULL;
} }
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) {
return NULL;
}
PyObject *req_type[3]; PyObject *req_type[3];
req_type[0] = state->Module_type; req_type[0] = state->Module_type;
req_type[1] = state->Expression_type; req_type[1] = state->Expression_type;
@ -10189,7 +10210,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
int PyAST_Check(PyObject* obj) int PyAST_Check(PyObject* obj)
{ {
astmodulestate *state = get_global_ast_state(); struct ast_state *state = get_ast_state();
if (state == NULL) { if (state == NULL) {
return -1; return -1;
} }