diff --git a/Include/pythonrun.h b/Include/pythonrun.h index b963a06af17..13ed4715e79 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -37,6 +37,7 @@ DL_IMPORT(PyObject *) PyRun_FileEx(FILE *, char *, int, PyObject *, PyObject *, int); DL_IMPORT(PyObject *) Py_CompileString(char *, char *, int); +DL_IMPORT(struct symtable *) Py_SymtableString(char *, char *, int); DL_IMPORT(void) PyErr_Print(void); DL_IMPORT(void) PyErr_PrintEx(int); diff --git a/Include/symtable.h b/Include/symtable.h new file mode 100644 index 00000000000..2d427aa6899 --- /dev/null +++ b/Include/symtable.h @@ -0,0 +1,99 @@ +#ifndef Py_SYMTABLE_H +#define Py_SYMTABLE_H +#ifdef __cplusplus +extern "C" { +#endif + +/* A symbol table is constructed each time PyNode_Compile() is + called. The table walks the entire parse tree and identifies each + use or definition of a variable. + + The symbol table contains a dictionary for each code block in a + module: The symbol dictionary for the block. They keys of these + dictionaries are the name of all variables used or defined in the + block; the integer values are used to store several flags, + e.g. DEF_PARAM indicates that a variable is a parameter to a + function. + + The slots st_cur_XXX pointers always refer to the current code + block. The st_cur slot is the symbol dictionary. The st_cur_id + slot is the id is the key in st_symbols. The st_cur_name slot is + the name of the current scope. The st_cur_type slot is one of + TYPE_FUNCTION, TYPE_CLASS, or TYPE_MODULE. The st_cur_children is + a list of the ids of the current node's children. + + The st_symbols slot is a dictionary that maps code block ids to + symbol dictionaries. The keys are generated by a counter that is + incremented each time a new code block is found. The counter is + identifies a specific scope, because both passes walk the parse + tree in the same order. + + The st_varnames slot is a dictionary that maps code block ids to + parameter lists. The st_global slot always refers to the symbol + dictionary for the module. + + The st_children slot is a dictionary that maps ids to a list + containing the ids of its children. + + If st_keep is true then the namespace info pushed on st_stack will + also be stored in st_scopes. This is useful if the symbol table is + being passed to something other than the compiler. +*/ + +struct symtable { + int st_pass; /* pass == 1 or 2 */ + int st_keep; /* true if symtable will be returned */ + PyObject *st_symbols; /* dictionary of symbol tables */ + PyObject *st_varnames; /* dictionary of parameter lists */ + PyObject *st_stack; /* stack of namespace info */ + PyObject *st_scopes; /* dictionary of namespace info */ + PyObject *st_children; /* dictionary (id=[ids]) */ + PyObject *st_cur; /* borrowed ref to dict in st_symbols */ + PyObject *st_cur_name; /* string, name of current scope */ + PyObject *st_cur_id; /* int id of current code block */ + PyObject *st_cur_children; /* ref to current children list */ + int st_cur_type; /* type of current scope */ + int st_cur_lineno; /* line number where current scope begins */ + PyObject *st_global; /* borrowed ref to MODULE in st_symbols */ + int st_nscopes; /* number of scopes */ + int st_errors; /* number of errors */ + char *st_private; /* name of current class or NULL */ + int st_tmpname; /* temporary name counter */ + int st_nested; /* bool (true if nested scope) */ +}; + +DL_IMPORT(struct symtable *) PyNode_CompileSymtable(struct _node *, char *); +DL_IMPORT(void) PySymtable_Free(struct symtable *); + + +#define TOP "global" +#define NOOPT ".noopt" + +/* Flags for def-use information */ + +#define DEF_GLOBAL 1 /* global stmt */ +#define DEF_LOCAL 2 /* assignment in code block */ +#define DEF_PARAM 2<<1 /* formal parameter */ +#define USE 2<<2 /* name is used */ +#define DEF_STAR 2<<3 /* parameter is star arg */ +#define DEF_DOUBLESTAR 2<<4 /* parameter is star-star arg */ +#define DEF_INTUPLE 2<<5 /* name defined in tuple in parameters */ +#define DEF_FREE 2<<6 /* name used by not defined in nested scope */ +#define DEF_FREE_GLOBAL 2<<7 /* free variable is actually implicit global */ +#define DEF_FREE_CLASS 2<<8 /* free variable from class's method */ +#define DEF_IMPORT 2<<9 /* assignment occurred via import */ + +#define TYPE_FUNCTION 1 +#define TYPE_CLASS 2 +#define TYPE_MODULE 3 + +#define LOCAL 1 +#define GLOBAL_EXPLICIT 2 +#define GLOBAL_IMPLICIT 3 +#define FREE 4 +#define CELL 5 + +#ifdef __cplusplus +} +#endif +#endif /* !Py_SYMTABLE_H */ diff --git a/Python/compile.c b/Python/compile.c index 3f12e419aa1..acad6673eab 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -16,6 +16,7 @@ #include "token.h" #include "graminit.h" #include "compile.h" +#include "symtable.h" #include "opcode.h" #include "structmember.h" @@ -45,16 +46,6 @@ int Py_OptimizeFlag = 0; #define VAR_STORE 1 #define VAR_DELETE 2 -#define TYPE_FUNCTION 1 -#define TYPE_CLASS 2 -#define TYPE_MODULE 3 - -#define LOCAL 1 -#define GLOBAL_EXPLICIT 2 -#define GLOBAL_IMPLICIT 3 -#define FREE 4 -#define CELL 5 - #define DEL_CLOSURE_ERROR \ "can not delete variable '%.400s' referenced in nested scope" @@ -367,74 +358,6 @@ struct compiling { struct symtable *c_symtable; /* pointer to module symbol table */ }; -/* A symbol table is constructed each time PyNode_Compile() is - called. The table walks the entire parse tree and identifies each - use or definition of a variable. - - The symbol table contains a dictionary for each code block in a - module: The symbol dictionary for the block. They keys of these - dictionaries are the name of all variables used or defined in the - block; the integer values are used to store several flags, - e.g. DEF_PARAM indicates that a variable is a parameter to a - function. - - The slots st_cur_XXX pointers always refer to the current code - block. The st_cur slot is the symbol dictionary. The st_cur_id - slot is the id is the key in st_symbols. The st_cur_name slot is - the name of the current scope. The st_cur_type slot is one of - TYPE_FUNCTION, TYPE_CLASS, or TYPE_MODULE. The st_cur_children is - a list of the ids of the current node's children. - - The st_symbols slot is a dictionary that maps code block ids to - symbol dictionaries. The keys are generated by a counter that is - incremented each time a new code block is found. The counter is - identifies a specific scope, because both passes walk the parse - tree in the same order. - - The st_varnames slot is a dictionary that maps code block ids to - parameter lists. The st_global slot always refers to the symbol - dictionary for the module. - - The st_children slot is a dictionary that maps ids to a list - containing the ids of its children. -*/ - -struct symtable { - int st_pass; /* pass == 1 or 2 */ - PyObject *st_symbols; /* dictionary of symbol tables */ - PyObject *st_varnames; /* dictionary of parameter lists */ - PyObject *st_stack; /* stack of namespace info */ - PyObject *st_children; /* dictionary (id=[ids]) */ - PyObject *st_cur; /* borrowed ref to dict in st_symbols */ - PyObject *st_cur_name; /* string, name of current scope */ - PyObject *st_cur_id; /* int id of current code block */ - PyObject *st_cur_children; /* ref to current children list */ - int st_cur_type; /* type of current scope */ - PyObject *st_global; /* borrowed ref to MODULE in st_symbols */ - int st_scopes; /* number of scopes */ - int st_errors; /* number of errors */ - char *st_private; /* name of current class or NULL */ - int st_tmpname; /* temporary name counter */ - int st_nested; /* bool (true if nested scope) */ -}; - -#define TOP "global" -#define NOOPT ".noopt" - -/* Flags for def-use information */ - -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define USE 2<<2 /* name is used */ -#define DEF_STAR 2<<3 /* parameter is star arg */ -#define DEF_DOUBLESTAR 2<<4 /* parameter is star-star arg */ -#define DEF_INTUPLE 2<<5 /* name defined in tuple in parameters */ -#define DEF_FREE 2<<6 /* name used by not defined in nested scope */ -#define DEF_FREE_GLOBAL 2<<7 /* free variable is actually implicit global */ -#define DEF_FREE_CLASS 2<<8 /* free variable from class's method */ -#define DEF_IMPORT 2<<9 /* assignment occurred via import */ - int is_free(int v) { if ((v & (USE | DEF_FREE)) @@ -553,9 +476,8 @@ static int get_ref_type(struct compiling *, char *); /* symtable operations */ static int symtable_build(struct compiling *, node *); static int symtable_load_symbols(struct compiling *); -static struct symtable *symtable_init(void); -static void symtable_free(struct symtable *); -static int symtable_enter_scope(struct symtable *, char *, int); +static struct symtable *symtable_init(int); +static void symtable_enter_scope(struct symtable *, char *, int, int); static int symtable_exit_scope(struct symtable *); static int symtable_update_cur(struct symtable *); static int symtable_add_def(struct symtable *, char *, int); @@ -2217,7 +2139,8 @@ com_test(struct compiling *c, node *n) PyObject *co; int i, closure; int ndefs = com_argdefs(c, CHILD(n, 0)); - symtable_enter_scope(c->c_symtable, "lambda", lambdef); + symtable_enter_scope(c->c_symtable, "lambda", lambdef, + n->n_lineno); co = (PyObject *) icompile(CHILD(n, 0), c); symtable_exit_scope(c->c_symtable); if (co == NULL) { @@ -3335,7 +3258,8 @@ com_funcdef(struct compiling *c, node *n) int ndefs; REQ(n, funcdef); /* funcdef: 'def' NAME parameters ':' suite */ ndefs = com_argdefs(c, n); - symtable_enter_scope(c->c_symtable, STR(CHILD(n, 1)), TYPE(n)); + symtable_enter_scope(c->c_symtable, STR(CHILD(n, 1)), TYPE(n), + n->n_lineno); co = (PyObject *)icompile(n, c); symtable_exit_scope(c->c_symtable); if (co == NULL) @@ -3395,7 +3319,7 @@ com_classdef(struct compiling *c, node *n) else com_bases(c, CHILD(n, 3)); name = STR(CHILD(n, 1)); - symtable_enter_scope(c->c_symtable, name, TYPE(n)); + symtable_enter_scope(c->c_symtable, name, TYPE(n), n->n_lineno); co = (PyObject *)icompile(n, c); symtable_exit_scope(c->c_symtable); if (co == NULL) @@ -3872,6 +3796,27 @@ PyNode_Compile(node *n, char *filename) return jcompile(n, filename, NULL); } +struct symtable * +PyNode_CompileSymtable(node *n, char *filename) +{ + struct symtable *st; + + st = symtable_init(1); + if (st == NULL) + return NULL; + symtable_enter_scope(st, TOP, TYPE(n), n->n_lineno); + if (st->st_errors > 0) { + PySymtable_Free(st); + return NULL; + } + symtable_node(st, n); + if (st->st_errors > 0) { + PySymtable_Free(st); + return NULL; + } + return st; +} + static PyCodeObject * icompile(node *n, struct compiling *base) { @@ -3948,7 +3893,7 @@ jcompile(node *n, char *filename, struct compiling *base) } exit: if (base == NULL) - symtable_free(sc.c_symtable); + PySymtable_Free(sc.c_symtable); com_free(&sc); return co; } @@ -4005,15 +3950,16 @@ get_ref_type(struct compiling *c, char *name) static int symtable_build(struct compiling *c, node *n) { - if ((c->c_symtable = symtable_init()) == NULL) + if ((c->c_symtable = symtable_init(0)) == NULL) return -1; - if (symtable_enter_scope(c->c_symtable, TOP, TYPE(n)) < 0) + symtable_enter_scope(c->c_symtable, TOP, TYPE(n), n->n_lineno); + if (c->c_symtable->st_errors > 0) return -1; symtable_node(c->c_symtable, n); if (c->c_symtable->st_errors > 0) return -1; /* reset for second pass */ - c->c_symtable->st_scopes = 1; + c->c_symtable->st_nscopes = 1; c->c_symtable->st_pass = 2; return 0; } @@ -4192,7 +4138,7 @@ symtable_load_symbols(struct compiling *c) } static struct symtable * -symtable_init() +symtable_init(int keep) { struct symtable *st; PyObject *d; @@ -4201,6 +4147,7 @@ symtable_init() if (st == NULL) return NULL; st->st_pass = 1; + st->st_keep = keep; if ((st->st_stack = PyList_New(0)) == NULL) goto fail; if ((st->st_symbols = PyDict_New()) == NULL) @@ -4214,41 +4161,49 @@ symtable_init() if (PyDict_SetItemString(st->st_symbols, TOP, d) < 0) goto fail; Py_DECREF(d); - st->st_global = d; + if (keep) { + if ((d = PyDict_New()) == NULL) + goto fail; + st->st_scopes = d; + } else + st->st_scopes = NULL; + st->st_global = d; /* use ref borrowed from st->st_symbols */ st->st_cur = NULL; st->st_cur_id = NULL; st->st_cur_name = NULL; st->st_cur_children = NULL; st->st_cur_type = 0; st->st_nested = 0; - st->st_scopes = 0; + st->st_nscopes = 0; st->st_errors = 0; st->st_tmpname = 0; st->st_private = NULL; return st; fail: - symtable_free(st); + PySymtable_Free(st); return NULL; } -static void -symtable_free(struct symtable *st) +void +PySymtable_Free(struct symtable *st) { Py_XDECREF(st->st_symbols); Py_XDECREF(st->st_varnames); Py_XDECREF(st->st_children); Py_XDECREF(st->st_stack); + Py_XDECREF(st->st_scopes); Py_XDECREF(st->st_cur_id); Py_XDECREF(st->st_cur_name); PyMem_Free((void *)st); } static PyObject * -make_scope_info(PyObject *id, PyObject *name, int nested, int type) +make_scope_info(PyObject *id, PyObject *name, int nested, int type, + int lineno) { - PyObject *t, *i1 = NULL, *i2 = NULL; + PyObject *t, *i1 = NULL, *i2 = NULL, *i3 = NULL; - t = PyTuple_New(4); + t = PyTuple_New(5); if (t == NULL) return NULL; i1 = PyInt_FromLong(nested); @@ -4257,6 +4212,9 @@ make_scope_info(PyObject *id, PyObject *name, int nested, int type) i2 = PyInt_FromLong(type); if (i2 == NULL) goto fail; + i3 = PyInt_FromLong(lineno); + if (i3 == NULL) + goto fail; Py_INCREF(name); Py_INCREF(id); @@ -4265,11 +4223,13 @@ make_scope_info(PyObject *id, PyObject *name, int nested, int type) /* i1 & i2 alloced here; don't need incref */ PyTuple_SET_ITEM(t, 2, i1); PyTuple_SET_ITEM(t, 3, i2); + PyTuple_SET_ITEM(t, 4, i3); return t; fail: Py_XDECREF(t); Py_XDECREF(i1); Py_XDECREF(i2); + Py_XDECREF(i3); return NULL; } @@ -4378,7 +4338,6 @@ symtable_undo_free(struct symtable *st, PyObject *id, return 0; } - static int symtable_exit_scope(struct symtable *st) { @@ -4402,26 +4361,40 @@ symtable_exit_scope(struct symtable *st) return symtable_update_cur(st); } -static int -symtable_enter_scope(struct symtable *st, char *name, int type) +static void +symtable_enter_scope(struct symtable *st, char *name, int type, + int lineno) { PyObject *o; if (st->st_cur) { /* push current scope info on stack */ o = make_scope_info(st->st_cur_id, st->st_cur_name, - st->st_nested, st->st_cur_type); - if (o == NULL) - return -1; + st->st_nested, st->st_cur_type, + st->st_cur_lineno); + if (o == NULL) { + st->st_errors++; + return; + } if (PyList_Append(st->st_stack, o) < 0) { Py_DECREF(o); - return -1; + st->st_errors++; + return; + } + if (st->st_keep) { + if (PyDict_SetItem(st->st_scopes, + st->st_cur_id, o) < 0) { + Py_DECREF(o); + st->st_errors++; + return; + } } Py_DECREF(o); } st->st_cur_name = PyString_FromString(name); if (st->st_nested || st->st_cur_type == TYPE_FUNCTION) st->st_nested = 1; + st->st_cur_lineno = lineno; switch (type) { case funcdef: case lambdef: @@ -4437,30 +4410,37 @@ symtable_enter_scope(struct symtable *st, char *name, int type) break; default: fprintf(stderr, "invalid symtable scope: %d\n", type); - return -1; + st->st_errors++; + return; } /* update st_cur_id and parent's st_cur_children */ - o = PyInt_FromLong(st->st_scopes++); - if (o == NULL) - return -1; + o = PyInt_FromLong(st->st_nscopes++); + if (o == NULL) { + st->st_errors++; + return; + } if (st->st_cur_children) { if (PyList_Append(st->st_cur_children, o) < 0) { Py_DECREF(o); - return -1; + st->st_errors++; + return; } } st->st_cur_id = o; /* create st_cur_children list */ o = PyList_New(0); - if (o == NULL) - return -1; + if (o == NULL) { + st->st_errors++; + return; + } if (PyDict_SetItem(st->st_children, st->st_cur_id, o) < 0) { Py_DECREF(o); - return -1; + st->st_errors++; + return; } Py_DECREF(o); - return symtable_update_cur(st); + symtable_update_cur(st); } static int @@ -4575,7 +4555,7 @@ symtable_node(struct symtable *st, node *n) char *func_name = STR(CHILD(n, 1)); symtable_add_def(st, func_name, DEF_LOCAL); symtable_default_args(st, CHILD(n, 2)); - symtable_enter_scope(st, func_name, TYPE(n)); + symtable_enter_scope(st, func_name, TYPE(n), n->n_lineno); symtable_funcdef(st, n); symtable_exit_scope(st); break; @@ -4583,7 +4563,7 @@ symtable_node(struct symtable *st, node *n) case lambdef: if (NCH(n) == 4) symtable_default_args(st, CHILD(n, 1)); - symtable_enter_scope(st, "lambda", TYPE(n)); + symtable_enter_scope(st, "lambda", TYPE(n), n->n_lineno); symtable_funcdef(st, n); symtable_exit_scope(st); break; @@ -4597,7 +4577,7 @@ symtable_node(struct symtable *st, node *n) symtable_node(st, CHILD(bases, i)); } } - symtable_enter_scope(st, class_name, TYPE(n)); + symtable_enter_scope(st, class_name, TYPE(n), n->n_lineno); tmp = st->st_private; st->st_private = class_name; symtable_node(st, CHILD(n, NCH(n) - 1)); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5f907978abb..5e0377d6ac8 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -9,6 +9,7 @@ #include "parsetok.h" #include "errcode.h" #include "compile.h" +#include "symtable.h" #include "eval.h" #include "marshal.h" @@ -963,6 +964,19 @@ Py_CompileString(char *str, char *filename, int start) return (PyObject *)co; } +struct symtable * +Py_SymtableString(char *str, char *filename, int start) +{ + node *n; + struct symtable *st; + n = PyParser_SimpleParseString(str, start); + if (n == NULL) + return NULL; + st = PyNode_CompileSymtable(n, filename); + PyNode_Free(n); + return st; +} + /* Simplified interface to parsefile -- return node or set exception */ node *