traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-41194: Pass module state in Python-ast.c (GH-21284) 

Rework asdl_c.py to pass the module state to functions in
Python-ast.c, instead of using astmodulestate_global.

Handle also PyState_AddModule() failure in init_types().
This commit is contained in:
Victor Stinner 2020-07-03 11:35:37 +02:00 committed by GitHub
parent 9d006977d7
commit 74419f0c64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 2072 additions and 2150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
Parser/asdl_c.py
View File

@ -387,7 +387,7 @@ class PickleVisitor(EmitVisitor):
class Obj2ModPrototypeVisitor(PickleVisitor): class Obj2ModPrototypeVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);" code = "static int obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena);"
self.emit(code % (name, get_c_type(name)), 0) self.emit(code % (name, get_c_type(name)), 0)
visitSum = visitProduct visitSum = visitProduct
@ -397,7 +397,7 @@ class Obj2ModVisitor(PickleVisitor):
def funcHeader(self, name): def funcHeader(self, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("int", 0) self.emit("int", 0)
self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("int isinstance;", 1) self.emit("int isinstance;", 1)
self.emit("", 0) self.emit("", 0)
@ -419,7 +419,7 @@ class Obj2ModVisitor(PickleVisitor):
self.funcHeader(name) self.funcHeader(name)
for t in sum.types: for t in sum.types:
line = ("isinstance = PyObject_IsInstance(obj, " line = ("isinstance = PyObject_IsInstance(obj, "
"astmodulestate_global->%s_type);") "state->%s_type);")
self.emit(line % (t.name,), 1) self.emit(line % (t.name,), 1)
self.emit("if (isinstance == -1) {", 1) self.emit("if (isinstance == -1) {", 1)
self.emit("return 1;", 2) self.emit("return 1;", 2)
@ -448,7 +448,7 @@ class Obj2ModVisitor(PickleVisitor):
for a in sum.attributes: for a in sum.attributes:
self.visitField(a, name, sum=sum, depth=1) self.visitField(a, name, sum=sum, depth=1)
for t in sum.types: for t in sum.types:
self.emit("tp = astmodulestate_global->%s_type;" % (t.name,), 1) self.emit("tp = state->%s_type;" % (t.name,), 1)
self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1) self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1)
self.emit("if (isinstance == -1) {", 1) self.emit("if (isinstance == -1) {", 1)
self.emit("return 1;", 2) self.emit("return 1;", 2)
@ -479,7 +479,7 @@ class Obj2ModVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("int", 0) self.emit("int", 0)
self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("PyObject* tmp = NULL;", 1) self.emit("PyObject* tmp = NULL;", 1)
for f in prod.fields: for f in prod.fields:
@ -525,7 +525,7 @@ class Obj2ModVisitor(PickleVisitor):
def visitField(self, field, name, sum=None, prod=None, depth=0): def visitField(self, field, name, sum=None, prod=None, depth=0):
ctype = get_c_type(field.type) ctype = get_c_type(field.type)
line = "if (_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {" line = "if (_PyObject_LookupAttr(obj, state->%s, &tmp) < 0) {"
self.emit(line % field.name, depth) self.emit(line % field.name, depth)
self.emit("return 1;", depth+1) self.emit("return 1;", depth+1)
self.emit("}", depth) self.emit("}", depth)
@ -568,7 +568,7 @@ class Obj2ModVisitor(PickleVisitor):
self.emit("%s val;" % ctype, depth+2) self.emit("%s val;" % ctype, depth+2)
self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2) self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2)
self.emit("Py_INCREF(tmp2);", depth+2) self.emit("Py_INCREF(tmp2);", depth+2)
self.emit("res = obj2ast_%s(tmp2, &val, arena);" % self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" %
field.type, depth+2, reflow=False) field.type, depth+2, reflow=False)
self.emit("Py_DECREF(tmp2);", depth+2) self.emit("Py_DECREF(tmp2);", depth+2)
self.emit("if (res != 0) goto failed;", depth+2) self.emit("if (res != 0) goto failed;", depth+2)
@ -582,7 +582,7 @@ class Obj2ModVisitor(PickleVisitor):
self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2) self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2)
self.emit("}", depth+1) self.emit("}", depth+1)
else: else:
self.emit("res = obj2ast_%s(tmp, &%s, arena);" % self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" %
(field.type, field.name), depth+1) (field.type, field.name), depth+1)
self.emit("if (res != 0) goto failed;", depth+1) self.emit("if (res != 0) goto failed;", depth+1)
@ -604,7 +604,7 @@ class PyTypesDeclareVisitor(PickleVisitor):
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
self.emit_type("%s_type" % name) self.emit_type("%s_type" % name)
self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) self.emit("static PyObject* ast2obj_%s(astmodulestate *state, void*);" % name, 0)
if prod.attributes: if prod.attributes:
for a in prod.attributes: for a in prod.attributes:
self.emit_identifier(a.name) self.emit_identifier(a.name)
@ -634,7 +634,7 @@ class PyTypesDeclareVisitor(PickleVisitor):
ptype = get_c_type(name) ptype = get_c_type(name)
for t in sum.types: for t in sum.types:
self.emit_singleton("%s_singleton" % t.name) self.emit_singleton("%s_singleton" % t.name)
self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) self.emit("static PyObject* ast2obj_%s(astmodulestate *state, %s);" % (name, ptype), 0)
for t in sum.types: for t in sum.types:
self.visitConstructor(t, name) self.visitConstructor(t, name)
@ -691,7 +691,8 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
Py_ssize_t i, numfields = 0; Py_ssize_t i, numfields = 0;
int res = -1; int res = -1;
PyObject *key, *value, *fields; PyObject *key, *value, *fields;
if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) { astmodulestate *state = astmodulestate_global;
if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
goto cleanup; goto cleanup;
} }
if (fields) { if (fields) {
@ -759,8 +760,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
static PyObject * static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused) ast_type_reduce(PyObject *self, PyObject *unused)
{ {
astmodulestate *state = astmodulestate_global;
PyObject *dict; PyObject *dict;
if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) { if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) {
return NULL; return NULL;
} }
if (dict) { if (dict) {
@ -809,7 +811,8 @@ static PyType_Spec AST_type_spec = {
}; };
static PyObject * static PyObject *
make_type(const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) make_type(astmodulestate *state, const char *type, PyObject* base,
const char* const* fields, int num_fields, const char *doc)
{ {
PyObject *fnames, *result; PyObject *fnames, *result;
int i; int i;
@ -825,16 +828,16 @@ make_type(const char *type, PyObject* base, const char* const* fields, int num_f
} }
result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOOOs}", result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOOOs}",
type, base, type, base,
astmodulestate_global->_fields, fnames, state->_fields, fnames,
astmodulestate_global->__module__, state->__module__,
astmodulestate_global->ast, state->ast,
astmodulestate_global->__doc__, doc); state->__doc__, doc);
Py_DECREF(fnames); Py_DECREF(fnames);
return result; return result;
} }
static int static int
add_attributes(PyObject *type, const char * const *attrs, int num_fields) add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields)
{ {
int i, result; int i, result;
PyObject *s, *l = PyTuple_New(num_fields); PyObject *s, *l = PyTuple_New(num_fields);
@ -848,14 +851,14 @@ add_attributes(PyObject *type, const char * const *attrs, int num_fields)
} }
PyTuple_SET_ITEM(l, i, s); PyTuple_SET_ITEM(l, i, s);
} }
result = PyObject_SetAttr(type, astmodulestate_global->_attributes, l) >= 0; result = PyObject_SetAttr(type, state->_attributes, l) >= 0;
Py_DECREF(l); Py_DECREF(l);
return result; return result;
} }
/* Conversion AST -> Python */ /* Conversion AST -> Python */
static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*))
{ {
Py_ssize_t i, n = asdl_seq_LEN(seq); Py_ssize_t i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n); PyObject *result = PyList_New(n);
@ -863,7 +866,7 @@ static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
if (!result) if (!result)
return NULL; return NULL;
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
value = func(asdl_seq_GET(seq, i)); value = func(state, asdl_seq_GET(seq, i));
if (!value) { if (!value) {
Py_DECREF(result); Py_DECREF(result);
return NULL; return NULL;
@ -873,7 +876,7 @@ static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
return result; return result;
} }
static PyObject* ast2obj_object(void *o) static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o)
{ {
if (!o) if (!o)
o = Py_None; o = Py_None;
@ -884,14 +887,14 @@ static PyObject* ast2obj_object(void *o)
#define ast2obj_identifier ast2obj_object #define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object #define ast2obj_string ast2obj_object
static PyObject* ast2obj_int(long b) static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b)
{ {
return PyLong_FromLong(b); return PyLong_FromLong(b);
} }
/* Conversion Python -> AST */ /* Conversion Python -> AST */
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (obj == Py_None) if (obj == Py_None)
obj = NULL; obj = NULL;
@ -906,7 +909,7 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
return 0; return 0;
} }
static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (PyArena_AddPyObject(arena, obj) < 0) { if (PyArena_AddPyObject(arena, obj) < 0) {
*out = NULL; *out = NULL;
@ -917,25 +920,25 @@ static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
return 0; return 0;
} }
static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && obj != Py_None) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str");
return 1; return 1;
} }
return obj2ast_object(obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena)
{ {
if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
return 1; return 1;
} }
return obj2ast_object(obj, out, arena); return obj2ast_object(state, obj, out, arena);
} }
static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena)
{ {
int i; int i;
if (!PyLong_Check(obj)) { if (!PyLong_Check(obj)) {
@ -950,13 +953,13 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
return 0; return 0;
} }
static int add_ast_fields(void) static int add_ast_fields(astmodulestate *state)
{ {
PyObject *empty_tuple; PyObject *empty_tuple;
empty_tuple = PyTuple_New(0); empty_tuple = PyTuple_New(0);
if (!empty_tuple || if (!empty_tuple ||
PyObject_SetAttrString(astmodulestate_global->AST_type, "_fields", empty_tuple) < 0 || PyObject_SetAttrString(state->AST_type, "_fields", empty_tuple) < 0 ||
PyObject_SetAttrString(astmodulestate_global->AST_type, "_attributes", empty_tuple) < 0) { PyObject_SetAttrString(state->AST_type, "_attributes", empty_tuple) < 0) {
Py_XDECREF(empty_tuple); Py_XDECREF(empty_tuple);
return -1; return -1;
} }
@ -968,18 +971,24 @@ static int add_ast_fields(void)
self.emit("static int init_types(void)",0) self.emit("static int init_types(void)",0)
self.emit("{", 0) self.emit("{", 0)
self.emit("PyObject *m;", 1) self.emit("PyObject *module = PyState_FindModule(&_astmodule);", 1)
self.emit("if (PyState_FindModule(&_astmodule) == NULL) {", 1) self.emit("if (module == NULL) {", 1)
self.emit("m = PyModule_Create(&_astmodule);", 2) self.emit("module = PyModule_Create(&_astmodule);", 2)
self.emit("if (!m) return 0;", 2) self.emit("if (!module) {", 2)
self.emit("PyState_AddModule(m, &_astmodule);", 2) self.emit("return 0;", 3)
self.emit("}", 2)
self.emit("if (PyState_AddModule(module, &_astmodule) < 0) {", 2)
self.emit("return 0;", 3)
self.emit("}", 2)
self.emit("}", 1) self.emit("}", 1)
self.emit("astmodulestate *state = astmodulestate_global;", 1) self.emit("", 0)
self.emit("astmodulestate *state = get_ast_state(module);", 1)
self.emit("if (state->initialized) return 1;", 1) self.emit("if (state->initialized) return 1;", 1)
self.emit("if (init_identifiers() < 0) return 0;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1)
self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
self.emit("if (!state->AST_type) return 0;", 1) self.emit("if (!state->AST_type) return 0;", 1)
self.emit("if (add_ast_fields() < 0) return 0;", 1) self.emit("if (add_ast_fields(state) < 0) return 0;", 1)
for dfn in mod.dfns: for dfn in mod.dfns:
self.visit(dfn) self.visit(dfn)
self.emit("state->initialized = 1;", 1) self.emit("state->initialized = 1;", 1)
@ -991,31 +1000,31 @@ static int add_ast_fields(void)
fields = name+"_fields" fields = name+"_fields"
else: else:
fields = "NULL" fields = "NULL"
self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d,' % self.emit('state->%s_type = make_type(state, "%s", state->AST_type, %s, %d,' %
(name, name, fields, len(prod.fields)), 1) (name, name, fields, len(prod.fields)), 1)
self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False) self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False)
self.emit("if (!state->%s_type) return 0;" % name, 1) self.emit("if (!state->%s_type) return 0;" % name, 1)
self.emit_type("AST_type") self.emit_type("AST_type")
self.emit_type("%s_type" % name) self.emit_type("%s_type" % name)
if prod.attributes: if prod.attributes:
self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" % self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" %
(name, name, len(prod.attributes)), 1) (name, name, len(prod.attributes)), 1)
else: else:
self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1) self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1)
self.emit_defaults(name, prod.fields, 1) self.emit_defaults(name, prod.fields, 1)
self.emit_defaults(name, prod.attributes, 1) self.emit_defaults(name, prod.attributes, 1)
def visitSum(self, sum, name): def visitSum(self, sum, name):
self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0,' % self.emit('state->%s_type = make_type(state, "%s", state->AST_type, NULL, 0,' %
(name, name), 1) (name, name), 1)
self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False) self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False)
self.emit_type("%s_type" % name) self.emit_type("%s_type" % name)
self.emit("if (!state->%s_type) return 0;" % name, 1) self.emit("if (!state->%s_type) return 0;" % name, 1)
if sum.attributes: if sum.attributes:
self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" % self.emit("if (!add_attributes(state, state->%s_type, %s_attributes, %d)) return 0;" %
(name, name, len(sum.attributes)), 1) (name, name, len(sum.attributes)), 1)
else: else:
self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1) self.emit("if (!add_attributes(state, state->%s_type, NULL, 0)) return 0;" % name, 1)
self.emit_defaults(name, sum.attributes, 1) self.emit_defaults(name, sum.attributes, 1)
simple = is_simple(sum) simple = is_simple(sum)
for t in sum.types: for t in sum.types:
@ -1026,7 +1035,7 @@ static int add_ast_fields(void)
fields = cons.name+"_fields" fields = cons.name+"_fields"
else: else:
fields = "NULL" fields = "NULL"
self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d,' % self.emit('state->%s_type = make_type(state, "%s", state->%s_type, %s, %d,' %
(cons.name, cons.name, name, fields, len(cons.fields)), 1) (cons.name, cons.name, name, fields, len(cons.fields)), 1)
self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False) self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False)
self.emit("if (!state->%s_type) return 0;" % cons.name, 1) self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
@ -1056,10 +1065,13 @@ class ASTModuleVisitor(PickleVisitor):
self.emit("if (!init_types()) return NULL;", 1) self.emit("if (!init_types()) return NULL;", 1)
self.emit('m = PyState_FindModule(&_astmodule);', 1) self.emit('m = PyState_FindModule(&_astmodule);', 1)
self.emit("if (!m) return NULL;", 1) self.emit("if (!m) return NULL;", 1)
self.emit('if (PyModule_AddObject(m, "AST", astmodulestate_global->AST_type) < 0) {', 1) self.emit('astmodulestate *state = get_ast_state(m);', 1)
self.emit('', 1)
self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1)
self.emit('goto error;', 2) self.emit('goto error;', 2)
self.emit('}', 1) self.emit('}', 1)
self.emit('Py_INCREF(astmodulestate(m)->AST_type);', 1) self.emit('Py_INCREF(state->AST_type);', 1)
self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) {', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) {', 1)
self.emit("goto error;", 2) self.emit("goto error;", 2)
self.emit('}', 1) self.emit('}', 1)
@ -1090,10 +1102,10 @@ class ASTModuleVisitor(PickleVisitor):
def addObj(self, name): def addObj(self, name):
self.emit("if (PyModule_AddObject(m, \"%s\", " self.emit("if (PyModule_AddObject(m, \"%s\", "
"astmodulestate_global->%s_type) < 0) {" % (name, name), 1) "state->%s_type) < 0) {" % (name, name), 1)
self.emit("goto error;", 2) self.emit("goto error;", 2)
self.emit('}', 1) self.emit('}', 1)
self.emit("Py_INCREF(astmodulestate(m)->%s_type);" % name, 1) self.emit("Py_INCREF(state->%s_type);" % name, 1)
_SPECIALIZED_SEQUENCES = ('stmt', 'expr') _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
@ -1127,7 +1139,7 @@ class ObjVisitor(PickleVisitor):
def func_begin(self, name): def func_begin(self, name):
ctype = get_c_type(name) ctype = get_c_type(name)
self.emit("PyObject*", 0) self.emit("PyObject*", 0)
self.emit("ast2obj_%s(void* _o)" % (name), 0) self.emit("ast2obj_%s(astmodulestate *state, void* _o)" % (name), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
self.emit("PyObject *result = NULL, *value = NULL;", 1) self.emit("PyObject *result = NULL, *value = NULL;", 1)
@ -1135,7 +1147,6 @@ class ObjVisitor(PickleVisitor):
self.emit('if (!o) {', 1) self.emit('if (!o) {', 1)
self.emit("Py_RETURN_NONE;", 2) self.emit("Py_RETURN_NONE;", 2)
self.emit("}", 1) self.emit("}", 1)
self.emit('', 0)
def func_end(self): def func_end(self):
self.emit("return result;", 1) self.emit("return result;", 1)
@ -1157,43 +1168,43 @@ class ObjVisitor(PickleVisitor):
self.visitConstructor(t, i + 1, name) self.visitConstructor(t, i + 1, name)
self.emit("}", 1) self.emit("}", 1)
for a in sum.attributes: for a in sum.attributes:
self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1)
self.emit("if (!value) goto failed;", 1) self.emit("if (!value) goto failed;", 1)
self.emit('if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)' % a.name, 1) self.emit('if (PyObject_SetAttr(result, state->%s, value) < 0)' % a.name, 1)
self.emit('goto failed;', 2) self.emit('goto failed;', 2)
self.emit('Py_DECREF(value);', 1) self.emit('Py_DECREF(value);', 1)
self.func_end() self.func_end()
def simpleSum(self, sum, name): def simpleSum(self, sum, name):
self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0) self.emit("PyObject* ast2obj_%s(astmodulestate *state, %s_ty o)" % (name, name), 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("switch(o) {", 1) self.emit("switch(o) {", 1)
for t in sum.types: for t in sum.types:
self.emit("case %s:" % t.name, 2) self.emit("case %s:" % t.name, 2)
self.emit("Py_INCREF(astmodulestate_global->%s_singleton);" % t.name, 3) self.emit("Py_INCREF(state->%s_singleton);" % t.name, 3)
self.emit("return astmodulestate_global->%s_singleton;" % t.name, 3) self.emit("return state->%s_singleton;" % t.name, 3)
self.emit("}", 1) self.emit("}", 1)
self.emit("Py_UNREACHABLE();", 1); self.emit("Py_UNREACHABLE();", 1);
self.emit("}", 0) self.emit("}", 0)
def visitProduct(self, prod, name): def visitProduct(self, prod, name):
self.func_begin(name) self.func_begin(name)
self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % name, 1) self.emit("tp = (PyTypeObject *)state->%s_type;" % name, 1)
self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 1); self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 1);
self.emit("if (!result) return NULL;", 1) self.emit("if (!result) return NULL;", 1)
for field in prod.fields: for field in prod.fields:
self.visitField(field, name, 1, True) self.visitField(field, name, 1, True)
for a in prod.attributes: for a in prod.attributes:
self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) self.emit("value = ast2obj_%s(state, o->%s);" % (a.type, a.name), 1)
self.emit("if (!value) goto failed;", 1) self.emit("if (!value) goto failed;", 1)
self.emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)" % a.name, 1) self.emit("if (PyObject_SetAttr(result, state->%s, value) < 0)" % a.name, 1)
self.emit('goto failed;', 2) self.emit('goto failed;', 2)
self.emit('Py_DECREF(value);', 1) self.emit('Py_DECREF(value);', 1)
self.func_end() self.func_end()
def visitConstructor(self, cons, enum, name): def visitConstructor(self, cons, enum, name):
self.emit("case %s_kind:" % cons.name, 1) self.emit("case %s_kind:" % cons.name, 1)
self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % cons.name, 2) self.emit("tp = (PyTypeObject *)state->%s_type;" % cons.name, 2)
self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 2); self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 2);
self.emit("if (!result) goto failed;", 2) self.emit("if (!result) goto failed;", 2)
for f in cons.fields: for f in cons.fields:
@ -1209,7 +1220,7 @@ class ObjVisitor(PickleVisitor):
value = "o->v.%s.%s" % (name, field.name) value = "o->v.%s.%s" % (name, field.name)
self.set(field, value, depth) self.set(field, value, depth)
emit("if (!value) goto failed;", 0) emit("if (!value) goto failed;", 0)
emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) == -1)" % field.name, 0) emit("if (PyObject_SetAttr(result, state->%s, value) == -1)" % field.name, 0)
emit("goto failed;", 1) emit("goto failed;", 1)
emit("Py_DECREF(value);", 0) emit("Py_DECREF(value);", 0)
@ -1237,14 +1248,14 @@ class ObjVisitor(PickleVisitor):
self.emit("if (!value) goto failed;", depth+1) self.emit("if (!value) goto failed;", depth+1)
self.emit("for(i = 0; i < n; i++)", depth+1) self.emit("for(i = 0; i < n; i++)", depth+1)
# This cannot fail, so no need for error handling # This cannot fail, so no need for error handling
self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value, self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop(state, (cmpop_ty)asdl_seq_GET(%s, i)));" % value,
depth+2, reflow=False) depth+2, reflow=False)
self.emit("}", depth) self.emit("}", depth)
else: else:
self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth) self.emit("value = ast2obj_list(state, %s, ast2obj_%s);" % (value, field.type), depth)
else: else:
ctype = get_c_type(field.type) ctype = get_c_type(field.type)
self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False) self.emit("value = ast2obj_%s(state, %s);" % (field.type, value), depth, reflow=False)
class PartingShots(StaticVisitor): class PartingShots(StaticVisitor):
@ -1254,13 +1265,13 @@ PyObject* PyAST_mod2obj(mod_ty t)
{ {
if (!init_types()) if (!init_types())
return NULL; return NULL;
return ast2obj_mod(t); astmodulestate *state = astmodulestate_global;
return ast2obj_mod(state, t);
} }
/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
{ {
PyObject *req_type[3];
const char * const req_name[] = {"Module", "Expression", "Interactive"}; const char * const req_name[] = {"Module", "Expression", "Interactive"};
int isinstance; int isinstance;
@ -1268,9 +1279,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return NULL; return NULL;
} }
req_type[0] = astmodulestate_global->Module_type; astmodulestate *state = astmodulestate_global;
req_type[1] = astmodulestate_global->Expression_type; PyObject *req_type[3];
req_type[2] = astmodulestate_global->Interactive_type; req_type[0] = state->Module_type;
req_type[1] = state->Expression_type;
req_type[2] = state->Interactive_type;
assert(0 <= mode && mode <= 2); assert(0 <= mode && mode <= 2);
@ -1287,7 +1300,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
} }
mod_ty res = NULL; mod_ty res = NULL;
if (obj2ast_mod(ast, &res, arena) != 0) if (obj2ast_mod(state, ast, &res, arena) != 0)
return NULL; return NULL;
else else
return res; return res;
@ -1297,7 +1310,8 @@ int PyAST_Check(PyObject* obj)
{ {
if (!init_types()) if (!init_types())
return -1; return -1;
return PyObject_IsInstance(obj, astmodulestate_global->AST_type); astmodulestate *state = astmodulestate_global;
return PyObject_IsInstance(obj, state->AST_type);
} }
""" """
@ -1347,22 +1361,31 @@ def generate_module_def(f, mod):
f.write(' PyObject *' + s + ';\n') f.write(' PyObject *' + s + ';\n')
f.write('} astmodulestate;\n\n') f.write('} astmodulestate;\n\n')
f.write(""" f.write("""
#define astmodulestate(o) ((astmodulestate *)PyModule_GetState(o)) static astmodulestate *
get_ast_state(PyObject *module)
{
assert(module != NULL);
void *state = PyModule_GetState(module);
assert(state != NULL);
return (astmodulestate *)state;
}
static int astmodule_clear(PyObject *module) static int astmodule_clear(PyObject *module)
{ {
astmodulestate *state = get_ast_state(module);
""") """)
for s in module_state: for s in module_state:
f.write(" Py_CLEAR(astmodulestate(module)->" + s + ');\n') f.write(" Py_CLEAR(state->" + s + ');\n')
f.write(""" f.write("""
return 0; return 0;
} }
static int astmodule_traverse(PyObject *module, visitproc visit, void* arg) static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
{ {
astmodulestate *state = get_ast_state(module);
""") """)
for s in module_state: for s in module_state:
f.write(" Py_VISIT(astmodulestate(module)->" + s + ');\n') f.write(" Py_VISIT(state->" + s + ');\n')
f.write(""" f.write("""
return 0; return 0;
} }
@ -1383,12 +1406,11 @@ static struct PyModuleDef _astmodule = {
astmodule_free, astmodule_free,
}; };
#define astmodulestate_global ((astmodulestate *)PyModule_GetState(PyState_FindModule(&_astmodule))) #define astmodulestate_global get_ast_state(PyState_FindModule(&_astmodule))
""") """)
f.write('static int init_identifiers(void)\n') f.write('static int init_identifiers(astmodulestate *state)\n')
f.write('{\n') f.write('{\n')
f.write(' astmodulestate *state = astmodulestate_global;\n')
for identifier in state_strings: for identifier in state_strings:
f.write(' if ((state->' + identifier) f.write(' if ((state->' + identifier)
f.write(' = PyUnicode_InternFromString("') f.write(' = PyUnicode_InternFromString("')

4040
Python/Python-ast.c generated
View File

File diff suppressed because it is too large Load Diff