From ee9b10a9872244df1ad576f3e398ad959bbab0fc Mon Sep 17 00:00:00 2001 From: Neal Norwitz Date: Mon, 31 Mar 2008 05:29:39 +0000 Subject: [PATCH] Merged revisions 61440-61441,61443,61445-61448,61451-61452,61455-61457,61459-61464,61466-61467,61469-61470,61476-61477,61479,61481-61482,61485,61487,61490,61493-61494,61497,61499-61502,61505-61506,61508,61511-61514,61519,61521-61522,61530-61531,61533-61537,61541-61555,61557-61558,61561-61562,61566-61569,61572-61574,61578-61579,61583-61584,61588-61589,61592,61594,61598-61601,61603-61604,61607-61612,61617,61619-61620,61624,61626,61628-61630,61635-61638,61640-61643,61645,61648,61653-61655,61659-61662,61664,61666,61668-61671,61673,61675,61679-61680,61682,61685-61686,61689-61695,61697-61699,61701-61703,61706,61710,61713,61717,61723,61726-61730,61736,61738,61740,61742,61745-61752,61754-61760,61762-61764,61768,61770-61772,61774-61775,61784-61787,61789-61792,61794-61795,61797-61806,61808-61809,61811-61812,61814-61819,61824,61826-61833,61835-61840,61843-61845,61848,61850,61854-61862,61865-61866,61868,61872-61873,61876-61877,61883-61888,61890-61891,61893-61899,61901-61903,61905-61912,61914,61917,61920-61921,61927,61930,61932-61934,61939,61941-61942,61944-61951,61955,61960-61963,61980,61982-61983,61991,61994-61996,62001-62003,62008-62010,62016-62017,62022,62024,62027,62031-62034,62041,62045-62046,62048,62050-62051,62055-62066,62068-62074,62076-62078 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r62048 | georg.brandl | 2008-03-29 23:53:55 -0700 (Sat, 29 Mar 2008) | 2 lines Adapt test_ast to the new ExceptHandler type. ........ r62050 | georg.brandl | 2008-03-30 00:09:22 -0700 (Sun, 30 Mar 2008) | 2 lines Convert test_ast to unittest and add a test for r62049. ........ r62051 | georg.brandl | 2008-03-30 12:00:49 -0700 (Sun, 30 Mar 2008) | 2 lines Make _fields attr for no fields consistent with _attributes attr. ........ r62059 | georg.brandl | 2008-03-30 13:20:39 -0700 (Sun, 30 Mar 2008) | 2 lines Make AST nodes pickleable. ........ --- Lib/test/test_ast.py | 113 +++++++++++++++++++++++++++++-------------- Parser/asdl_c.py | 40 +++++++++++---- Python/Python-ast.c | 44 ++++++++++++----- 3 files changed, 140 insertions(+), 57 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d08003236f7..f4fbc94f5a2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,4 +1,5 @@ -import sys +import sys, unittest +from test import test_support import _ast def to_tuple(t): @@ -15,6 +16,7 @@ def to_tuple(t): result.append(to_tuple(getattr(t, f))) return tuple(result) + # These tests are compiled through "exec" # There should be atleast one test per statement exec_tests = [ @@ -112,42 +114,83 @@ eval_tests = [ # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension # excepthandler, arguments, keywords, alias -if __name__=='__main__' and sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for s in statements: - print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") - print("]") - print("run_tests()") - raise SystemExit +class AST_Tests(unittest.TestCase): -def test_order(ast_node, parent_pos): + def _assert_order(self, ast_node, parent_pos): + if not isinstance(ast_node, _ast.AST) or ast_node._fields is None: + return + if isinstance(ast_node, (_ast.expr, _ast.stmt, _ast.excepthandler)): + node_pos = (ast_node.lineno, ast_node.col_offset) + self.assert_(node_pos >= parent_pos) + parent_pos = (ast_node.lineno, ast_node.col_offset) + for name in ast_node._fields: + value = getattr(ast_node, name) + if isinstance(value, list): + for child in value: + self._assert_order(child, parent_pos) + elif value is not None: + self._assert_order(value, parent_pos) - if not isinstance(ast_node, _ast.AST) or ast_node._fields is None: + def test_snippets(self): + for input, output, kind in ((exec_tests, exec_results, "exec"), + (single_tests, single_results, "single"), + (eval_tests, eval_results, "eval")): + for i, o in zip(input, output): + ast_tree = compile(i, "?", kind, _ast.PyCF_ONLY_AST) + self.assertEquals(to_tuple(ast_tree), o) + self._assert_order(ast_tree, (0, 0)) + + def test_nodeclasses(self): + x = _ast.BinOp(1, 2, 3, lineno=0) + self.assertEquals(x.left, 1) + self.assertEquals(x.op, 2) + self.assertEquals(x.right, 3) + self.assertEquals(x.lineno, 0) + + # node raises exception when not given enough arguments + self.assertRaises(TypeError, _ast.BinOp, 1, 2) + + # can set attributes through kwargs too + x = _ast.BinOp(left=1, op=2, right=3, lineno=0) + self.assertEquals(x.left, 1) + self.assertEquals(x.op, 2) + self.assertEquals(x.right, 3) + self.assertEquals(x.lineno, 0) + + # this used to fail because Sub._fields was None + x = _ast.Sub() + + def test_pickling(self): + import pickle + mods = [pickle] + try: + import cPickle + mods.append(cPickle) + except ImportError: + pass + protocols = [0, 1, 2] + for mod in mods: + for protocol in protocols: + for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): + ast2 = mod.loads(mod.dumps(ast, protocol)) + self.assertEquals(to_tuple(ast2), to_tuple(ast)) + +def test_main(): + test_support.run_unittest(AST_Tests) + +def main(): + if __name__ != '__main__': return - if isinstance(ast_node, (_ast.expr, _ast.stmt, _ast.excepthandler)): - node_pos = (ast_node.lineno, ast_node.col_offset) - assert node_pos >= parent_pos, (node_pos, parent_pos) - parent_pos = (ast_node.lineno, ast_node.col_offset) - for name in ast_node._fields: - value = getattr(ast_node, name) - if isinstance(value, list): - for child in value: - test_order(child, parent_pos) - elif value is not None: - test_order(value, parent_pos) - -def run_tests(): - for input, output, kind in ((exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval")): - for i, o in zip(input, output): - ast_tree = compile(i, "?", kind, 0x400) - tup = to_tuple(ast_tree) - assert tup == o, ("kind=%r\ninput=%r\nexpected=%r\ngot=%r" % - (kind, i, o, tup)) - test_order(ast_tree, (0, 0)) + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for s in statements: + print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") + print("]") + print("main()") + raise SystemExit + test_main() #### EVERYTHING BELOW IS GENERATED ##### exec_results = [ @@ -194,4 +237,4 @@ eval_results = [ ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))), ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)), ] -run_tests() +main() diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a38c2686bf5..1c83f507849 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -628,9 +628,34 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) return res; } +/* Pickling support */ +static PyObject * +ast_type_reduce(PyObject *self, PyObject *unused) +{ + PyObject *res; + PyObject *dict = PyObject_GetAttrString(self, "__dict__"); + if (dict == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) + PyErr_Clear(); + else + return NULL; + } + if (dict) { + res = Py_BuildValue("O()O", Py_TYPE(self), dict); + Py_DECREF(dict); + return res; + } + return Py_BuildValue("O()", Py_TYPE(self)); +} + +static PyMethodDef ast_type_methods[] = { + {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, + {NULL} +}; + static PyTypeObject AST_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "AST", + "_ast.AST", sizeof(PyObject), 0, 0, /* tp_dealloc */ @@ -656,7 +681,7 @@ static PyTypeObject AST_type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + ast_type_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -675,14 +700,9 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int { PyObject *fnames, *result; int i; - if (num_fields) { - fnames = PyTuple_New(num_fields); - if (!fnames) return NULL; - } else { - fnames = Py_None; - Py_INCREF(Py_None); - } - for(i=0; i < num_fields; i++) { + fnames = PyTuple_New(num_fields); + if (!fnames) return NULL; + for (i = 0; i < num_fields; i++) { PyObject *field = PyUnicode_FromString(fields[i]); if (!field) { Py_DECREF(fnames); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a50435f2067..9e9ba063962 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2,7 +2,7 @@ /* - __version__ 57783. + __version__ 62078. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -445,9 +445,34 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) return res; } +/* Pickling support */ +static PyObject * +ast_type_reduce(PyObject *self, PyObject *unused) +{ + PyObject *res; + PyObject *dict = PyObject_GetAttrString(self, "__dict__"); + if (dict == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) + PyErr_Clear(); + else + return NULL; + } + if (dict) { + res = Py_BuildValue("O()O", Py_TYPE(self), dict); + Py_DECREF(dict); + return res; + } + return Py_BuildValue("O()", Py_TYPE(self)); +} + +static PyMethodDef ast_type_methods[] = { + {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, + {NULL} +}; + static PyTypeObject AST_type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "AST", + "_ast.AST", sizeof(PyObject), 0, 0, /* tp_dealloc */ @@ -473,7 +498,7 @@ static PyTypeObject AST_type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + ast_type_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -492,14 +517,9 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int { PyObject *fnames, *result; int i; - if (num_fields) { - fnames = PyTuple_New(num_fields); - if (!fnames) return NULL; - } else { - fnames = Py_None; - Py_INCREF(Py_None); - } - for(i=0; i < num_fields; i++) { + fnames = PyTuple_New(num_fields); + if (!fnames) return NULL; + for (i = 0; i < num_fields; i++) { PyObject *field = PyUnicode_FromString(fields[i]); if (!field) { Py_DECREF(fnames); @@ -6375,7 +6395,7 @@ init_ast(void) if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return; - if (PyModule_AddStringConstant(m, "__version__", "57783") < 0) + if (PyModule_AddStringConstant(m, "__version__", "62078") < 0) return; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0)