Merged revisions 61440-61441,61443,61445-61448,61451-61452,61455-61457,61459-61464,61466-61467,61469-61470,61476-61477,61479,61481-61482,61485,61487,61490,61493-61494,61497,61499-61502,61505-61506,61508,61511-61514,61519,61521-61522,61530-61531,61533-61537,61541-61555,61557-61558,61561-61562,61566-61569,61572-61574,61578-61579,61583-61584,61588-61589,61592,61594,61598-61601,61603-61604,61607-61612,61617,61619-61620,61624,61626,61628-61630,61635-61638,61640-61643,61645,61648,61653-61655,61659-61662,61664,61666,61668-61671,61673,61675,61679-61680,61682,61685-61686,61689-61695,61697-61699,61701-61703,61706,61710,61713,61717,61723,61726-61730,61736,61738,61740,61742,61745-61752,61754-61760,61762-61764,61768,61770-61772,61774-61775,61784-61787,61789-61792,61794-61795,61797-61806,61808-61809,61811-61812,61814-61819,61824,61826-61833,61835-61840,61843-61845,61848,61850,61854-61862,61865-61866,61868,61872-61873,61876-61877,61883-61888,61890-61891,61893-61899,61901-61903,61905-61912,61914,61917,61920-61921,61927,61930,61932-61934,61939,61941-61942,61944-61951,61955,61960-61963,61980,61982-61983,61991,61994-61996,62001-62003,62008-62010,62016-62017,62022,62024,62027,62031-62034,62041,62045-62046,62048,62050-62051,62055-62066,62068-62074,62076-62078 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r62048 | georg.brandl | 2008-03-29 23:53:55 -0700 (Sat, 29 Mar 2008) | 2 lines

  Adapt test_ast to the new ExceptHandler type.
........
  r62050 | georg.brandl | 2008-03-30 00:09:22 -0700 (Sun, 30 Mar 2008) | 2 lines

  Convert test_ast to unittest and add a test for r62049.
........
  r62051 | georg.brandl | 2008-03-30 12:00:49 -0700 (Sun, 30 Mar 2008) | 2 lines

  Make _fields attr for no fields consistent with _attributes attr.
........
  r62059 | georg.brandl | 2008-03-30 13:20:39 -0700 (Sun, 30 Mar 2008) | 2 lines

  Make AST nodes pickleable.
........

Lib/test/test_ast.py

@@ -1,4 +1,5 @@
-import sys
+import sys, unittest
+from test import test_support
 import _ast
 
 def to_tuple(t):
@@ -15,6 +16,7 @@ def to_tuple(t):
         result.append(to_tuple(getattr(t, f)))
     return tuple(result)
 
+
 # These tests are compiled through "exec"
 # There should be atleast one test per statement
 exec_tests = [
@@ -112,42 +114,83 @@ eval_tests = [
 # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
 # excepthandler, arguments, keywords, alias
 
-if __name__=='__main__' and sys.argv[1:] == ['-g']:
+class AST_Tests(unittest.TestCase):
-    for statements, kind in ((exec_tests, "exec"), (single_tests, "single"),
-                             (eval_tests, "eval")):
-        print(kind+"_results = [")
-        for s in statements:
-            print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",")
-        print("]")
-    print("run_tests()")
-    raise SystemExit
 
-def test_order(ast_node, parent_pos):
+    def _assert_order(self, ast_node, parent_pos):
+        if not isinstance(ast_node, _ast.AST) or ast_node._fields is None:
+            return
+        if isinstance(ast_node, (_ast.expr, _ast.stmt, _ast.excepthandler)):
+            node_pos = (ast_node.lineno, ast_node.col_offset)
+            self.assert_(node_pos >= parent_pos)
+            parent_pos = (ast_node.lineno, ast_node.col_offset)
+        for name in ast_node._fields:
+            value = getattr(ast_node, name)
+            if isinstance(value, list):
+                for child in value:
+                    self._assert_order(child, parent_pos)
+            elif value is not None:
+                self._assert_order(value, parent_pos)
 
-    if not isinstance(ast_node, _ast.AST) or ast_node._fields is None:
+    def test_snippets(self):
+        for input, output, kind in ((exec_tests, exec_results, "exec"),
+                                    (single_tests, single_results, "single"),
+                                    (eval_tests, eval_results, "eval")):
+            for i, o in zip(input, output):
+                ast_tree = compile(i, "?", kind, _ast.PyCF_ONLY_AST)
+                self.assertEquals(to_tuple(ast_tree), o)
+                self._assert_order(ast_tree, (0, 0))
+
+    def test_nodeclasses(self):
+        x = _ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+        self.assertEquals(x.lineno, 0)
+
+        # node raises exception when not given enough arguments
+        self.assertRaises(TypeError, _ast.BinOp, 1, 2)
+
+        # can set attributes through kwargs too
+        x = _ast.BinOp(left=1, op=2, right=3, lineno=0)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+        self.assertEquals(x.lineno, 0)
+
+        # this used to fail because Sub._fields was None
+        x = _ast.Sub()
+
+    def test_pickling(self):
+        import pickle
+        mods = [pickle]
+        try:
+            import cPickle
+            mods.append(cPickle)
+        except ImportError:
+            pass
+        protocols = [0, 1, 2]
+        for mod in mods:
+            for protocol in protocols:
+                for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
+                    ast2 = mod.loads(mod.dumps(ast, protocol))
+                    self.assertEquals(to_tuple(ast2), to_tuple(ast))
+
+def test_main():
+    test_support.run_unittest(AST_Tests)
+
+def main():
+    if __name__ != '__main__':
         return
-    if isinstance(ast_node, (_ast.expr, _ast.stmt, _ast.excepthandler)):
+    if sys.argv[1:] == ['-g']:
-        node_pos = (ast_node.lineno, ast_node.col_offset)
+        for statements, kind in ((exec_tests, "exec"), (single_tests, "single"),
-        assert node_pos >= parent_pos, (node_pos, parent_pos)
+                                 (eval_tests, "eval")):
-        parent_pos = (ast_node.lineno, ast_node.col_offset)
+            print(kind+"_results = [")
-    for name in ast_node._fields:
+            for s in statements:
-        value = getattr(ast_node, name)
+                print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",")
-        if isinstance(value, list):
+            print("]")
-            for child in value:
+        print("main()")
-                test_order(child, parent_pos)
+        raise SystemExit
-        elif value is not None:
+    test_main()
-            test_order(value, parent_pos)
-
-def run_tests():
-    for input, output, kind in ((exec_tests, exec_results, "exec"),
-                                (single_tests, single_results, "single"),
-                                (eval_tests, eval_results, "eval")):
-        for i, o in zip(input, output):
-            ast_tree = compile(i, "?", kind, 0x400)
-            tup = to_tuple(ast_tree)
-            assert tup == o, ("kind=%r\ninput=%r\nexpected=%r\ngot=%r" %
-                              (kind, i, o, tup))
-            test_order(ast_tree, (0, 0))
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
@@ -194,4 +237,4 @@ eval_results = [
 ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
 ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
-run_tests()
+main()

Parser/asdl_c.py

@@ -628,9 +628,34 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
     return res;
 }
 
+/* Pickling support */
+static PyObject *
+ast_type_reduce(PyObject *self, PyObject *unused)
+{
+    PyObject *res;
+    PyObject *dict = PyObject_GetAttrString(self, "__dict__");
+    if (dict == NULL) {
+        if (PyErr_ExceptionMatches(PyExc_AttributeError))
+            PyErr_Clear();
+        else
+            return NULL;
+    }
+    if (dict) {
+        res = Py_BuildValue("O()O", Py_TYPE(self), dict);
+        Py_DECREF(dict);
+        return res;
+    }
+    return Py_BuildValue("O()", Py_TYPE(self));
+}
+
+static PyMethodDef ast_type_methods[] = {
+    {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
+    {NULL}
+};
+
 static PyTypeObject AST_type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
-    "AST",
+    "_ast.AST",
     sizeof(PyObject),
     0,
     0,                       /* tp_dealloc */
@@ -656,7 +681,7 @@ static PyTypeObject AST_type = {
     0,                       /* tp_weaklistoffset */
     0,                       /* tp_iter */
     0,                       /* tp_iternext */
-    0,                       /* tp_methods */
+    ast_type_methods,        /* tp_methods */
     0,                       /* tp_members */
     0,                       /* tp_getset */
     0,                       /* tp_base */
@@ -675,14 +700,9 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
 {
     PyObject *fnames, *result;
     int i;
-    if (num_fields) {
+    fnames = PyTuple_New(num_fields);
-        fnames = PyTuple_New(num_fields);
+    if (!fnames) return NULL;
-        if (!fnames) return NULL;
+    for (i = 0; i < num_fields; i++) {
-    } else {
-        fnames = Py_None;
-        Py_INCREF(Py_None);
-    }
-    for(i=0; i < num_fields; i++) {
         PyObject *field = PyUnicode_FromString(fields[i]);
         if (!field) {
             Py_DECREF(fnames);

Python/Python-ast.c

@@ -2,7 +2,7 @@
 
 
 /*
-   __version__ 57783.
+   __version__ 62078.
 
    This module must be committed separately after each AST grammar change;
    The __version__ number is set to the revision number of the commit
@@ -445,9 +445,34 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
     return res;
 }
 
+/* Pickling support */
+static PyObject *
+ast_type_reduce(PyObject *self, PyObject *unused)
+{
+    PyObject *res;
+    PyObject *dict = PyObject_GetAttrString(self, "__dict__");
+    if (dict == NULL) {
+        if (PyErr_ExceptionMatches(PyExc_AttributeError))
+            PyErr_Clear();
+        else
+            return NULL;
+    }
+    if (dict) {
+        res = Py_BuildValue("O()O", Py_TYPE(self), dict);
+        Py_DECREF(dict);
+        return res;
+    }
+    return Py_BuildValue("O()", Py_TYPE(self));
+}
+
+static PyMethodDef ast_type_methods[] = {
+    {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
+    {NULL}
+};
+
 static PyTypeObject AST_type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
-    "AST",
+    "_ast.AST",
     sizeof(PyObject),
     0,
     0,                       /* tp_dealloc */
@@ -473,7 +498,7 @@ static PyTypeObject AST_type = {
     0,                       /* tp_weaklistoffset */
     0,                       /* tp_iter */
     0,                       /* tp_iternext */
-    0,                       /* tp_methods */
+    ast_type_methods,        /* tp_methods */
     0,                       /* tp_members */
     0,                       /* tp_getset */
     0,                       /* tp_base */
@@ -492,14 +517,9 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
 {
     PyObject *fnames, *result;
     int i;
-    if (num_fields) {
+    fnames = PyTuple_New(num_fields);
-        fnames = PyTuple_New(num_fields);
+    if (!fnames) return NULL;
-        if (!fnames) return NULL;
+    for (i = 0; i < num_fields; i++) {
-    } else {
-        fnames = Py_None;
-        Py_INCREF(Py_None);
-    }
-    for(i=0; i < num_fields; i++) {
         PyObject *field = PyUnicode_FromString(fields[i]);
         if (!field) {
             Py_DECREF(fnames);
@@ -6375,7 +6395,7 @@ init_ast(void)
         if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;
         if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)
                 return;
-        if (PyModule_AddStringConstant(m, "__version__", "57783") < 0)
+        if (PyModule_AddStringConstant(m, "__version__", "62078") < 0)
                 return;
         if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return;
         if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0)