Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-41194: The _ast module cannot be loaded more than once (GH-21290) 

Fix a crash in the _ast module: it can no longer be loaded more than
once. It now uses a global state rather than a module state.

* Move _ast module state: use a global state instead.
* Set _astmodule.m_size to -1, so the extension cannot be loaded more
  than once.
This commit is contained in:
Victor Stinner 2020-07-03 14:15:53 +02:00 committed by GitHub
parent 74419f0c64
commit 91e1bc18bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 66 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS.d/next/Library/2020-07-03-13-15-08.bpo-41194.djrKjs.rst Normal file
View File

@ -0,0 +1,2 @@
Fix a crash in the ``_ast`` module: it can no longer be loaded more than once.
It now uses a global state rather than a module state.

71
Parser/asdl_c.py
View File

@ -691,7 +691,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
Py_ssize_t i, numfields = 0; Py_ssize_t i, numfields = 0;
int res = -1; int res = -1;
PyObject *key, *value, *fields; PyObject *key, *value, *fields;
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
goto cleanup; goto cleanup;
} }
@ -760,7 +760,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
static PyObject * static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused) ast_type_reduce(PyObject *self, PyObject *unused)
{ {
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
PyObject *dict; PyObject *dict;
if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) {
return NULL; return NULL;
@ -971,19 +971,7 @@ static int add_ast_fields(astmodulestate *state)
self.emit("static int init_types(void)",0) self.emit("static int init_types(void)",0)
self.emit("{", 0) self.emit("{", 0)
self.emit("PyObject *module = PyState_FindModule(&_astmodule);", 1) self.emit("astmodulestate *state = get_global_ast_state();", 1)
self.emit("if (module == NULL) {", 1)
self.emit("module = PyModule_Create(&_astmodule);", 2)
self.emit("if (!module) {", 2)
self.emit("return 0;", 3)
self.emit("}", 2)
self.emit("if (PyState_AddModule(module, &_astmodule) < 0) {", 2)
self.emit("return 0;", 3)
self.emit("}", 2)
self.emit("}", 1)
self.emit("", 0)
self.emit("astmodulestate *state = get_ast_state(module);", 1)
self.emit("if (state->initialized) return 1;", 1) self.emit("if (state->initialized) return 1;", 1)
self.emit("if (init_identifiers(state) < 0) return 0;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1)
self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
@ -1061,13 +1049,16 @@ class ASTModuleVisitor(PickleVisitor):
self.emit("PyMODINIT_FUNC", 0) self.emit("PyMODINIT_FUNC", 0)
self.emit("PyInit__ast(void)", 0) self.emit("PyInit__ast(void)", 0)
self.emit("{", 0) self.emit("{", 0)
self.emit("PyObject *m;", 1) self.emit("PyObject *m = PyModule_Create(&_astmodule);", 1)
self.emit("if (!init_types()) return NULL;", 1) self.emit("if (!m) {", 1)
self.emit('m = PyState_FindModule(&_astmodule);', 1) self.emit("return NULL;", 2)
self.emit("if (!m) return NULL;", 1) self.emit("}", 1)
self.emit('astmodulestate *state = get_ast_state(m);', 1) self.emit('astmodulestate *state = get_ast_state(m);', 1)
self.emit('', 1) self.emit('', 1)
self.emit("if (!init_types()) {", 1)
self.emit("goto error;", 2)
self.emit("}", 1)
self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1)
self.emit('goto error;', 2) self.emit('goto error;', 2)
self.emit('}', 1) self.emit('}', 1)
@ -1084,6 +1075,7 @@ class ASTModuleVisitor(PickleVisitor):
for dfn in mod.dfns: for dfn in mod.dfns:
self.visit(dfn) self.visit(dfn)
self.emit("return m;", 1) self.emit("return m;", 1)
self.emit("", 0)
self.emit("error:", 0) self.emit("error:", 0)
self.emit("Py_DECREF(m);", 1) self.emit("Py_DECREF(m);", 1)
self.emit("return NULL;", 1) self.emit("return NULL;", 1)
@ -1263,9 +1255,11 @@ class PartingShots(StaticVisitor):
CODE = """ CODE = """
PyObject* PyAST_mod2obj(mod_ty t) PyObject* PyAST_mod2obj(mod_ty t)
{ {
if (!init_types()) if (!init_types()) {
return NULL; return NULL;
astmodulestate *state = astmodulestate_global; }
astmodulestate *state = get_global_ast_state();
return ast2obj_mod(state, t); return ast2obj_mod(state, t);
} }
@ -1279,7 +1273,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return NULL; return NULL;
} }
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
PyObject *req_type[3]; PyObject *req_type[3];
req_type[0] = state->Module_type; req_type[0] = state->Module_type;
req_type[1] = state->Expression_type; req_type[1] = state->Expression_type;
@ -1287,8 +1281,9 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
assert(0 <= mode && mode <= 2); assert(0 <= mode && mode <= 2);
if (!init_types()) if (!init_types()) {
return NULL; return NULL;
}
isinstance = PyObject_IsInstance(ast, req_type[mode]); isinstance = PyObject_IsInstance(ast, req_type[mode]);
if (isinstance == -1) if (isinstance == -1)
@ -1308,9 +1303,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
int PyAST_Check(PyObject* obj) int PyAST_Check(PyObject* obj)
{ {
if (!init_types()) if (!init_types()) {
return -1; return -1;
astmodulestate *state = astmodulestate_global; }
astmodulestate *state = get_global_ast_state();
return PyObject_IsInstance(obj, state->AST_type); return PyObject_IsInstance(obj, state->AST_type);
} }
""" """
@ -1361,13 +1358,12 @@ def generate_module_def(f, mod):
f.write(' PyObject *' + s + ';\n') f.write(' PyObject *' + s + ';\n')
f.write('} astmodulestate;\n\n') f.write('} astmodulestate;\n\n')
f.write(""" f.write("""
static astmodulestate global_ast_state;
static astmodulestate * static astmodulestate *
get_ast_state(PyObject *module) get_ast_state(PyObject *Py_UNUSED(module))
{ {
assert(module != NULL); return &global_ast_state;
void *state = PyModule_GetState(module);
assert(state != NULL);
return (astmodulestate *)state;
} }
static int astmodule_clear(PyObject *module) static int astmodule_clear(PyObject *module)
@ -1396,17 +1392,14 @@ static void astmodule_free(void* module) {
static struct PyModuleDef _astmodule = { static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT, PyModuleDef_HEAD_INIT,
"_ast", .m_name = "_ast",
NULL, .m_size = -1,
sizeof(astmodulestate), .m_traverse = astmodule_traverse,
NULL, .m_clear = astmodule_clear,
NULL, .m_free = astmodule_free,
astmodule_traverse,
astmodule_clear,
astmodule_free,
}; };
#define astmodulestate_global get_ast_state(PyState_FindModule(&_astmodule)) #define get_global_ast_state() (&global_ast_state)
""") """)
f.write('static int init_identifiers(astmodulestate *state)\n') f.write('static int init_identifiers(astmodulestate *state)\n')

70
Python/Python-ast.c generated
View File

@ -224,13 +224,12 @@ typedef struct {
} astmodulestate; } astmodulestate;
static astmodulestate global_ast_state;
static astmodulestate * static astmodulestate *
get_ast_state(PyObject *module) get_ast_state(PyObject *Py_UNUSED(module))
{ {
assert(module != NULL); return &global_ast_state;
void *state = PyModule_GetState(module);
assert(state != NULL);
return (astmodulestate *)state;
} }
static int astmodule_clear(PyObject *module) static int astmodule_clear(PyObject *module)
@ -679,17 +678,14 @@ static void astmodule_free(void* module) {
static struct PyModuleDef _astmodule = { static struct PyModuleDef _astmodule = {
PyModuleDef_HEAD_INIT, PyModuleDef_HEAD_INIT,
"_ast", .m_name = "_ast",
NULL, .m_size = -1,
sizeof(astmodulestate), .m_traverse = astmodule_traverse,
NULL, .m_clear = astmodule_clear,
NULL, .m_free = astmodule_free,
astmodule_traverse,
astmodule_clear,
astmodule_free,
}; };
#define astmodulestate_global get_ast_state(PyState_FindModule(&_astmodule)) #define get_global_ast_state() (&global_ast_state)
static int init_identifiers(astmodulestate *state) static int init_identifiers(astmodulestate *state)
{ {
@ -1135,7 +1131,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
Py_ssize_t i, numfields = 0; Py_ssize_t i, numfields = 0;
int res = -1; int res = -1;
PyObject *key, *value, *fields; PyObject *key, *value, *fields;
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
goto cleanup; goto cleanup;
} }
@ -1204,7 +1200,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
static PyObject * static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused) ast_type_reduce(PyObject *self, PyObject *unused)
{ {
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
PyObject *dict; PyObject *dict;
if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) { if (_PyObject_LookupAttr(self, state->__dict__, &dict) < 0) {
return NULL; return NULL;
@ -1414,18 +1410,7 @@ static int add_ast_fields(astmodulestate *state)
static int init_types(void) static int init_types(void)
{ {
PyObject *module = PyState_FindModule(&_astmodule); astmodulestate *state = get_global_ast_state();
if (module == NULL) {
module = PyModule_Create(&_astmodule);
if (!module) {
return 0;
}
if (PyState_AddModule(module, &_astmodule) < 0) {
return 0;
}
}
astmodulestate *state = get_ast_state(module);
if (state->initialized) return 1; if (state->initialized) return 1;
if (init_identifiers(state) < 0) return 0; if (init_identifiers(state) < 0) return 0;
state->AST_type = PyType_FromSpec(&AST_type_spec); state->AST_type = PyType_FromSpec(&AST_type_spec);
@ -9857,12 +9842,15 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out,
PyMODINIT_FUNC PyMODINIT_FUNC
PyInit__ast(void) PyInit__ast(void)
{ {
PyObject *m; PyObject *m = PyModule_Create(&_astmodule);
if (!init_types()) return NULL; if (!m) {
m = PyState_FindModule(&_astmodule); return NULL;
if (!m) return NULL; }
astmodulestate *state = get_ast_state(m); astmodulestate *state = get_ast_state(m);
if (!init_types()) {
goto error;
}
if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {
goto error; goto error;
} }
@ -10303,6 +10291,7 @@ PyInit__ast(void)
} }
Py_INCREF(state->TypeIgnore_type); Py_INCREF(state->TypeIgnore_type);
return m; return m;
error: error:
Py_DECREF(m); Py_DECREF(m);
return NULL; return NULL;
@ -10311,9 +10300,11 @@ error:
PyObject* PyAST_mod2obj(mod_ty t) PyObject* PyAST_mod2obj(mod_ty t)
{ {
if (!init_types()) if (!init_types()) {
return NULL; return NULL;
astmodulestate *state = astmodulestate_global; }
astmodulestate *state = get_global_ast_state();
return ast2obj_mod(state, t); return ast2obj_mod(state, t);
} }
@ -10327,7 +10318,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return NULL; return NULL;
} }
astmodulestate *state = astmodulestate_global; astmodulestate *state = get_global_ast_state();
PyObject *req_type[3]; PyObject *req_type[3];
req_type[0] = state->Module_type; req_type[0] = state->Module_type;
req_type[1] = state->Expression_type; req_type[1] = state->Expression_type;
@ -10335,8 +10326,9 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
assert(0 <= mode && mode <= 2); assert(0 <= mode && mode <= 2);
if (!init_types()) if (!init_types()) {
return NULL; return NULL;
}
isinstance = PyObject_IsInstance(ast, req_type[mode]); isinstance = PyObject_IsInstance(ast, req_type[mode]);
if (isinstance == -1) if (isinstance == -1)
@ -10356,9 +10348,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
int PyAST_Check(PyObject* obj) int PyAST_Check(PyObject* obj)
{ {
if (!init_types()) if (!init_types()) {
return -1; return -1;
astmodulestate *state = astmodulestate_global; }
astmodulestate *state = get_global_ast_state();
return PyObject_IsInstance(obj, state->AST_type); return PyObject_IsInstance(obj, state->AST_type);
} }