From 207c9f3c5c0c41f329e7b7a2aeb8f2f7579dcbc9 Mon Sep 17 00:00:00 2001 From: Neal Norwitz Date: Mon, 31 Mar 2008 04:42:11 +0000 Subject: [PATCH] Merged revisions 62049,62054 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r62049 | georg.brandl | 2008-03-30 00:01:47 -0700 (Sun, 30 Mar 2008) | 2 lines #2505: allow easier creation of AST nodes. ........ r62054 | georg.brandl | 2008-03-30 12:43:27 -0700 (Sun, 30 Mar 2008) | 2 lines Fix error message -- "expects either 0 or 0 arguments" ........ --- Doc/library/_ast.rst | 27 ++++++++- Parser/asdl_c.py | 110 ++++++++++++++++++++++++++++++++--- Python/Python-ast.c | 135 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 241 insertions(+), 31 deletions(-) diff --git a/Doc/library/_ast.rst b/Doc/library/_ast.rst index 518798e6c41..cb7e44ecf58 100644 --- a/Doc/library/_ast.rst +++ b/Doc/library/_ast.rst @@ -44,9 +44,32 @@ node. The utf8 offset is recorded because the parser uses utf8 internally. If these attributes are marked as optional in the grammar (using a question mark), the value might be ``None``. If the attributes can have zero-or-more values (marked with an asterisk), the values are represented as Python lists. +All possible attributes must be present and have valid values when compiling an +AST with :func:`compile`. + +The constructor of a class ``_ast.T`` parses their arguments as follows: + +* If there are positional arguments, there must be as many as there are items in + ``T._fields``; they will be assigned as attributes of these names. +* If there are keyword arguments, they will set the attributes of the same names + to the given values. + +For example, to create and populate a ``UnaryOp`` node, you could use :: + + node = _ast.UnaryOp() + node.op = _ast.USub() + node.operand = _ast.Num() + node.operand.n = 5 + node.operand.lineno = 0 + node.operand.col_offset = 0 + node.lineno = 0 + node.col_offset = 0 + +or the more compact :: + + node = _ast.UnaryOp(_ast.USub(), _ast.Num(5, lineno=0, col_offset=0), + lineno=0, col_offset=0) -The constructors of all ``_ast`` classes don't take arguments; instead, if you -create instances, you must assign the required attributes separately. Abstract Grammar diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 1b5e18fd651..a38c2686bf5 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -577,6 +577,100 @@ class PyTypesVisitor(PickleVisitor): def visitModule(self, mod): self.emit(""" +static int +ast_type_init(PyObject *self, PyObject *args, PyObject *kw) +{ + Py_ssize_t i, numfields = 0; + int res = -1; + PyObject *key, *value, *fields; + fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); + if (!fields) + PyErr_Clear(); + if (fields) { + numfields = PySequence_Size(fields); + if (numfields == -1) + goto cleanup; + } + res = 0; /* if no error occurs, this stays 0 to the end */ + if (PyTuple_GET_SIZE(args) > 0) { + if (numfields != PyTuple_GET_SIZE(args)) { + PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" + "%" PY_FORMAT_SIZE_T "d positional argument%s", + Py_TYPE(self)->tp_name, + numfields == 0 ? "" : "either 0 or ", + numfields, numfields == 1 ? "" : "s"); + res = -1; + goto cleanup; + } + for (i = 0; i < PyTuple_GET_SIZE(args); i++) { + /* cannot be reached when fields is NULL */ + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + res = -1; + goto cleanup; + } + res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + Py_DECREF(name); + if (res < 0) + goto cleanup; + } + } + if (kw) { + i = 0; /* needed by PyDict_Next */ + while (PyDict_Next(kw, &i, &key, &value)) { + res = PyObject_SetAttr(self, key, value); + if (res < 0) + goto cleanup; + } + } + cleanup: + Py_XDECREF(fields); + return res; +} + +static PyTypeObject AST_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "AST", + sizeof(PyObject), + 0, + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ast_type_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) { PyObject *fnames, *result; @@ -605,7 +699,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) { int i, result; - PyObject *s, *l = PyList_New(num_fields); + PyObject *s, *l = PyTuple_New(num_fields); if (!l) return 0; for(i = 0; i < num_fields; i++) { s = PyUnicode_FromString(attrs[i]); @@ -613,7 +707,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) Py_DECREF(l); return 0; } - PyList_SET_ITEM(l, i, s); + PyTuple_SET_ITEM(l, i, s); } result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; Py_DECREF(l); @@ -696,7 +790,6 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) self.emit("{", 0) self.emit("static int initialized;", 1) self.emit("if (initialized) return 1;", 1) - self.emit('AST_type = make_type("AST", &PyBaseObject_Type, NULL, 0);', 1) for dfn in mod.dfns: self.visit(dfn) self.emit("initialized = 1;", 1) @@ -708,12 +801,13 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) fields = name.value+"_fields" else: fields = "NULL" - self.emit('%s_type = make_type("%s", AST_type, %s, %d);' % + self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % (name, name, fields, len(prod.fields)), 1) self.emit("if (!%s_type) return 0;" % name, 1) def visitSum(self, sum, name): - self.emit('%s_type = make_type("%s", AST_type, NULL, 0);' % (name, name), 1) + self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % + (name, name), 1) self.emit("if (!%s_type) return 0;" % name, 1) if sum.attributes: self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % @@ -752,7 +846,7 @@ class ASTModuleVisitor(PickleVisitor): self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1) self.emit("if (!m) return;", 1) self.emit("d = PyModule_GetDict(m);", 1) - self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return;', 1) + self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1) self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1) self.emit("return;", 2) # Value of version: "$Revision$" @@ -959,7 +1053,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { init_types(); - return PyObject_IsInstance(obj, (PyObject*)AST_type); + return PyObject_IsInstance(obj, (PyObject*)&AST_type); } """ @@ -1016,7 +1110,7 @@ def main(srcfile): f.write('#include "Python.h"\n') f.write('#include "%s-ast.h"\n' % mod.name) f.write('\n') - f.write("static PyTypeObject* AST_type;\n") + f.write("static PyTypeObject AST_type;\n") v = ChainOfVisitors( PyTypesDeclareVisitor(f), PyTypesVisitor(f), diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 9f855818172..b88b310af53 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12,7 +12,7 @@ #include "Python.h" #include "Python-ast.h" -static PyTypeObject* AST_type; +static PyTypeObject AST_type; static PyTypeObject *mod_type; static PyObject* ast2obj_mod(void*); static PyTypeObject *Module_type; @@ -391,6 +391,100 @@ static char *alias_fields[]={ }; +static int +ast_type_init(PyObject *self, PyObject *args, PyObject *kw) +{ + Py_ssize_t i, numfields = 0; + int res = -1; + PyObject *key, *value, *fields; + fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); + if (!fields) + PyErr_Clear(); + if (fields) { + numfields = PySequence_Size(fields); + if (numfields == -1) + goto cleanup; + } + res = 0; /* if no error occurs, this stays 0 to the end */ + if (PyTuple_GET_SIZE(args) > 0) { + if (numfields != PyTuple_GET_SIZE(args)) { + PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" + "%" PY_FORMAT_SIZE_T "d positional argument%s", + Py_TYPE(self)->tp_name, + numfields == 0 ? "" : "either 0 or ", + numfields, numfields == 1 ? "" : "s"); + res = -1; + goto cleanup; + } + for (i = 0; i < PyTuple_GET_SIZE(args); i++) { + /* cannot be reached when fields is NULL */ + PyObject *name = PySequence_GetItem(fields, i); + if (!name) { + res = -1; + goto cleanup; + } + res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); + Py_DECREF(name); + if (res < 0) + goto cleanup; + } + } + if (kw) { + i = 0; /* needed by PyDict_Next */ + while (PyDict_Next(kw, &i, &key, &value)) { + res = PyObject_SetAttr(self, key, value); + if (res < 0) + goto cleanup; + } + } + cleanup: + Py_XDECREF(fields); + return res; +} + +static PyTypeObject AST_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "AST", + sizeof(PyObject), + 0, + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ast_type_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) { PyObject *fnames, *result; @@ -419,7 +513,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) { int i, result; - PyObject *s, *l = PyList_New(num_fields); + PyObject *s, *l = PyTuple_New(num_fields); if (!l) return 0; for(i = 0; i < num_fields; i++) { s = PyUnicode_FromString(attrs[i]); @@ -427,7 +521,7 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) Py_DECREF(l); return 0; } - PyList_SET_ITEM(l, i, s); + PyTuple_SET_ITEM(l, i, s); } result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; Py_DECREF(l); @@ -509,8 +603,7 @@ static int init_types(void) { static int initialized; if (initialized) return 1; - AST_type = make_type("AST", &PyBaseObject_Type, NULL, 0); - mod_type = make_type("mod", AST_type, NULL, 0); + mod_type = make_type("mod", &AST_type, NULL, 0); if (!mod_type) return 0; if (!add_attributes(mod_type, NULL, 0)) return 0; Module_type = make_type("Module", mod_type, Module_fields, 1); @@ -523,7 +616,7 @@ static int init_types(void) if (!Expression_type) return 0; Suite_type = make_type("Suite", mod_type, Suite_fields, 1); if (!Suite_type) return 0; - stmt_type = make_type("stmt", AST_type, NULL, 0); + stmt_type = make_type("stmt", &AST_type, NULL, 0); if (!stmt_type) return 0; if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, @@ -573,7 +666,7 @@ static int init_types(void) if (!Break_type) return 0; Continue_type = make_type("Continue", stmt_type, NULL, 0); if (!Continue_type) return 0; - expr_type = make_type("expr", AST_type, NULL, 0); + expr_type = make_type("expr", &AST_type, NULL, 0); if (!expr_type) return 0; if (!add_attributes(expr_type, expr_attributes, 2)) return 0; BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2); @@ -625,7 +718,7 @@ static int init_types(void) if (!List_type) return 0; Tuple_type = make_type("Tuple", expr_type, Tuple_fields, 2); if (!Tuple_type) return 0; - expr_context_type = make_type("expr_context", AST_type, NULL, 0); + expr_context_type = make_type("expr_context", &AST_type, NULL, 0); if (!expr_context_type) return 0; if (!add_attributes(expr_context_type, NULL, 0)) return 0; Load_type = make_type("Load", expr_context_type, NULL, 0); @@ -652,7 +745,7 @@ static int init_types(void) if (!Param_type) return 0; Param_singleton = PyType_GenericNew(Param_type, NULL, NULL); if (!Param_singleton) return 0; - slice_type = make_type("slice", AST_type, NULL, 0); + slice_type = make_type("slice", &AST_type, NULL, 0); if (!slice_type) return 0; if (!add_attributes(slice_type, NULL, 0)) return 0; Slice_type = make_type("Slice", slice_type, Slice_fields, 3); @@ -661,7 +754,7 @@ static int init_types(void) if (!ExtSlice_type) return 0; Index_type = make_type("Index", slice_type, Index_fields, 1); if (!Index_type) return 0; - boolop_type = make_type("boolop", AST_type, NULL, 0); + boolop_type = make_type("boolop", &AST_type, NULL, 0); if (!boolop_type) return 0; if (!add_attributes(boolop_type, NULL, 0)) return 0; And_type = make_type("And", boolop_type, NULL, 0); @@ -672,7 +765,7 @@ static int init_types(void) if (!Or_type) return 0; Or_singleton = PyType_GenericNew(Or_type, NULL, NULL); if (!Or_singleton) return 0; - operator_type = make_type("operator", AST_type, NULL, 0); + operator_type = make_type("operator", &AST_type, NULL, 0); if (!operator_type) return 0; if (!add_attributes(operator_type, NULL, 0)) return 0; Add_type = make_type("Add", operator_type, NULL, 0); @@ -723,7 +816,7 @@ static int init_types(void) if (!FloorDiv_type) return 0; FloorDiv_singleton = PyType_GenericNew(FloorDiv_type, NULL, NULL); if (!FloorDiv_singleton) return 0; - unaryop_type = make_type("unaryop", AST_type, NULL, 0); + unaryop_type = make_type("unaryop", &AST_type, NULL, 0); if (!unaryop_type) return 0; if (!add_attributes(unaryop_type, NULL, 0)) return 0; Invert_type = make_type("Invert", unaryop_type, NULL, 0); @@ -742,7 +835,7 @@ static int init_types(void) if (!USub_type) return 0; USub_singleton = PyType_GenericNew(USub_type, NULL, NULL); if (!USub_singleton) return 0; - cmpop_type = make_type("cmpop", AST_type, NULL, 0); + cmpop_type = make_type("cmpop", &AST_type, NULL, 0); if (!cmpop_type) return 0; if (!add_attributes(cmpop_type, NULL, 0)) return 0; Eq_type = make_type("Eq", cmpop_type, NULL, 0); @@ -785,19 +878,19 @@ static int init_types(void) if (!NotIn_type) return 0; NotIn_singleton = PyType_GenericNew(NotIn_type, NULL, NULL); if (!NotIn_singleton) return 0; - comprehension_type = make_type("comprehension", AST_type, + comprehension_type = make_type("comprehension", &AST_type, comprehension_fields, 3); if (!comprehension_type) return 0; - excepthandler_type = make_type("excepthandler", AST_type, + excepthandler_type = make_type("excepthandler", &AST_type, excepthandler_fields, 5); if (!excepthandler_type) return 0; - arguments_type = make_type("arguments", AST_type, arguments_fields, 8); + arguments_type = make_type("arguments", &AST_type, arguments_fields, 8); if (!arguments_type) return 0; - arg_type = make_type("arg", AST_type, arg_fields, 2); + arg_type = make_type("arg", &AST_type, arg_fields, 2); if (!arg_type) return 0; - keyword_type = make_type("keyword", AST_type, keyword_fields, 2); + keyword_type = make_type("keyword", &AST_type, keyword_fields, 2); if (!keyword_type) return 0; - alias_type = make_type("alias", AST_type, alias_fields, 2); + alias_type = make_type("alias", &AST_type, alias_fields, 2); if (!alias_type) return 0; initialized = 1; return 1; @@ -6253,7 +6346,7 @@ init_ast(void) m = Py_InitModule3("_ast", NULL, NULL); if (!m) return; d = PyModule_GetDict(m); - if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return; + if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return; if (PyModule_AddStringConstant(m, "__version__", "57783") < 0) @@ -6440,7 +6533,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { init_types(); - return PyObject_IsInstance(obj, (PyObject*)AST_type); + return PyObject_IsInstance(obj, (PyObject*)&AST_type); }