Generate code to recursively copy an AST into

a tree of Python objects. Expose this through compile().

Include/Python-ast.h

@@ -404,3 +404,4 @@ arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
 keyword_ty keyword(identifier arg, expr_ty value, PyArena *arena);
 alias_ty alias(identifier name, identifier asname, PyArena *arena);
 
+PyObject* PyAST_mod2obj(mod_ty t);

Include/pythonrun.h

@@ -11,6 +11,7 @@ extern "C" {
 #define PyCF_MASK_OBSOLETE (CO_NESTED)
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
+#define PyCF_ONLY_AST 0x0400
 
 typedef struct {
 	int cf_flags;  /* bitmask of CO_xxx flags relevant to future */

Lib/test/test_ast.py

@@ -0,0 +1,179 @@
+import sys, itertools
+
+def to_tuple(t):
+    if t is None or isinstance(t, (str, int, long)):
+        return t
+    elif isinstance(t, list):
+        return [to_tuple(e) for e in t]
+    result = [t.__class__.__name__]
+    if t._fields is None:
+        return tuple(result)
+    for f in t._fields:
+        result.append(to_tuple(getattr(t, f)))
+    return tuple(result)
+    
+# These tests are compiled through "exec"
+# There should be atleast one test per statement
+exec_tests = [
+    # FunctionDef
+    "def f(): pass",
+    # ClassDef
+    "class C:pass",
+    # Return
+    "def f():return 1",
+    # Delete
+    "del v",
+    # Assign
+    "v = 1",
+    # AugAssign
+    "v += 1",
+    # Print
+    "print >>f, 1, ",
+    # For
+    "for v in v:pass",
+    # While
+    "while v:pass",
+    # If
+    "if v:pass",
+    # Raise
+    "raise Exception, 'string'",
+    # TryExcept
+    "try:\n  pass\nexcept Exception:\n  pass",
+    # TryFinally
+    "try:\n  pass\nfinally:\n  pass",
+    # Assert
+    "assert v",
+    # Import
+    "import sys",
+    # ImportFrom
+    "from sys import v",
+    # Exec
+    "exec 'v'",
+    # Global
+    "global v",
+    # Expr
+    "1",
+    # Pass,
+    "pass",
+    # Break
+    "break",
+    # Continue
+    "continue",
+]
+
+# These are compiled through "single"
+# because of overlap with "eval", it just tests what
+# can't be tested with "eval"
+single_tests = [
+    "1+2"
+]
+
+# These are compiled through "eval"
+# It should test all expressions
+eval_tests = [
+  # BoolOp
+  "a and b",
+  # BinOp
+  "a + b",
+  # UnaryOp
+  "not v",
+  # Lambda
+  "lambda:None",
+  # Dict
+  "{ 1:2 }",
+  # ListComp
+  "[a for b in c if d]",
+  # GeneratorExp
+  "(a for b in c if d)",
+  # Yield
+  #"def f():yield 3",
+  # Compare
+  "1 < 2 < 3",
+  # Call
+  "f(1,2,c=3,*d,**e)",
+  # Repr
+  "`v`",
+  # Num
+  "10L",
+  # Str
+  "'string'",
+  # Attribute
+  "a.b",
+  # Subscript
+  "a[b:c]",
+  # Name
+  "v",
+  # List
+  "[1,2,3]",
+  # Tuple
+  "1,2,3"
+]
+
+# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
+# excepthandler, arguments, keywords, alias
+
+if __name__=='__main__' and sys.argv[1:] == ['-g']:
+    for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")):
+        print kind+"_results = ["
+        for s in statements:
+            print repr(to_tuple(compile(s, "?", kind, 0x400)))+","
+        print "]"
+    print "run_tests()"
+    raise SystemExit
+
+def run_tests():
+    for input, output, kind in ((exec_tests, exec_results, "exec"), 
+                                        (single_tests, single_results, "single"), 
+                                        (eval_tests, eval_results, "eval")):
+        for i, o in itertools.izip(input, output):
+            assert to_tuple(compile(i, "?", kind, 0x400)) == o
+
+#### EVERYTHING BELOW IS GENERATED ##### 
+exec_results = [
+('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
+('Module', [('ClassDef', 'C', [], [('Pass',)])]),
+('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
+('Module', [('Delete', [('Name', 'v', ('Del',))])]),
+('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
+('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
+('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
+('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
+('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
+('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
+('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
+('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
+('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
+('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
+('Module', [('Import', [('alias', 'sys', None)])]),
+('Module', [('ImportFrom', 'sys', [('alias', 'v', None)])]),
+('Module', [('Exec', ('Str', 'v'), None, None)]),
+('Module', [('Global', ['v'])]),
+('Module', [('Expr', ('Num', 1))]),
+('Module', [('Pass',)]),
+('Module', [('Break',)]),
+('Module', [('Continue',)]),
+]
+single_results = [
+('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
+]
+eval_results = [
+('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
+('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
+('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
+('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
+('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
+('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
+('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
+('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
+('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
+('Expression', ('Repr', ('Name', 'v', ('Load',)))),
+('Expression', ('Num', 10L)),
+('Expression', ('Str', 'string')),
+('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
+('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Name', 'v', ('Load',))),
+('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
+('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
+]
+run_tests()
+

Misc/NEWS

@@ -74,7 +74,9 @@ Core and builtins
 
 - Speed up some Unicode operations.
 
-- A new AST parser implementation was completed.
+- A new AST parser implementation was completed. The abstract
+  syntax tree is available for read-only (non-compile) access
+  to Python code.
 
 - SF bug #1167751: fix incorrect code being for generator expressions.
   The following code now raises a SyntaxError:  foo(a = i for i in range(10))

Parser/asdl_c.py

@@ -344,13 +344,127 @@ class MarshalPrototypeVisitor(PickleVisitor):
 
     visitProduct = visitSum = prototype
 
-class FreePrototypeVisitor(PickleVisitor):
+class PyTypesDeclareVisitor(PickleVisitor):
 
     def prototype(self, sum, name):
         ctype = get_c_type(name)
         self.emit("void free_%s(%s);" % (name, ctype), 0)
 
-    visitProduct = visitSum = prototype
+    def visitProduct(self, prod, name):
+        self.emit("PyTypeObject *%s_type;" % name, 0)
+        self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
+        self.emit("char *%s_fields[]={" % name,0)
+        for f in prod.fields:
+            self.emit('"%s",' % f.name, 1)
+        self.emit("};", 0)
+        
+    def visitSum(self, sum, name):
+        self.emit("PyTypeObject *%s_type;" % name, 0)
+        ptype = "void*"
+        if is_simple(sum):
+            ptype = get_c_type(name)
+            tnames = []
+            for t in sum.types:
+                tnames.append(str(t.name)+"_singleton")
+            tnames = ", *".join(tnames)
+            self.emit("static PyObject *%s;" % tnames, 0)
+        self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
+        for t in sum.types:
+            self.visitConstructor(t, name)
+            
+    def visitConstructor(self, cons, name):
+        self.emit("PyTypeObject *%s_type;" % cons.name, 0)
+        self.emit("char *%s_fields[]={" % cons.name, 0)
+        for t in cons.fields:
+            self.emit('"%s",' % t.name, 1)
+        self.emit("};",0)
+
+class PyTypesVisitor(PickleVisitor):
+
+    def visitModule(self, mod):
+        self.emit("""
+static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
+{
+    PyObject *fnames, *result;
+    int i;
+    if (num_fields) {
+        fnames = PyTuple_New(num_fields);
+        if (!fnames) return NULL;
+    } else {
+        fnames = Py_None;
+        Py_INCREF(Py_None);
+    }
+    for(i=0; i < num_fields; i++) {
+        PyObject *field = PyString_FromString(fields[i]);
+        if (!field) {
+            Py_DECREF(fnames);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(fnames, i, field);
+    }
+    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sO}", type, base, "_fields", fnames);
+    Py_DECREF(fnames);
+    return (PyTypeObject*)result;
+}
+
+static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
+{
+    int i, n = asdl_seq_LEN(seq);
+    PyObject *result = PyList_New(n);
+    PyObject *value;
+    if (!result)
+        return NULL;
+    for (i = 0; i < n; i++) {
+        value = func(asdl_seq_GET(seq, i));
+        if (!value) {
+            Py_DECREF(result);
+            return NULL;
+        }
+        PyList_SET_ITEM(result, i, value);
+    }
+    return result;
+}
+
+static PyObject* ast2obj_object(void *o)
+{
+    if (!o)
+        o = Py_None;
+    Py_INCREF((PyObject*)o);
+    return (PyObject*)o;
+}
+#define ast2obj_identifier ast2obj_object
+#define ast2obj_string ast2obj_object
+static PyObject* ast2obj_bool(bool b)
+{
+    return PyBool_FromLong(b);
+}
+""", 0, reflow=False)
+
+        self.emit("static int initialized;", 0)
+        self.emit("static int init_types(void)",0)
+        self.emit("{", 0)
+        self.emit("if (initialized) return 1;", 1)
+        for dfn in mod.dfns:
+            self.visit(dfn)
+        self.emit("return 1;", 0);
+        self.emit("}", 0)
+
+    def visitProduct(self, prod, name):
+        self.emit('%s_type = make_type("%s", &PyBaseObject_Type, %s_fields, %d);' % 
+                        (name, name, name, len(prod.fields)), 1)
+        
+    def visitSum(self, sum, name):
+        self.emit('%s_type = make_type("%s", &PyBaseObject_Type, NULL, 0);' % (name, name), 1)
+        simple = is_simple(sum)
+        for t in sum.types:
+            self.visitConstructor(t, name, simple)
+            
+    def visitConstructor(self, cons, name, simple):
+        self.emit('%s_type = make_type("%s", %s_type, %s_fields, %d);' % 
+                            (cons.name, cons.name, name, cons.name, len(cons.fields)), 1)
+        if simple:
+            self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
+                             (cons.name, cons.name), 1)
 
 _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
 
@@ -400,48 +514,70 @@ free_seq_stmts(asdl_seq *seq)
 }
 '''
 
-class FreeVisitor(PickleVisitor):
+class ObjVisitor(PickleVisitor):
 
     def func_begin(self, name, has_seq):
         ctype = get_c_type(name)
-        self.emit("void", 0)
+        self.emit("PyObject*", 0)
-        self.emit("free_%s(%s o)" % (name, ctype), 0)
+        self.emit("ast2obj_%s(void* _o)" % (name), 0)
         self.emit("{", 0)
-        if has_seq:
+        self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
-            self.emit("int i, n;", 1)
+        self.emit("PyObject *result = NULL, *value = NULL;", 1)
-            self.emit("asdl_seq *seq;", 1)
+        self.emit('if (!o) {', 1)
-            self.emit('', 0)
+        self.emit("Py_INCREF(Py_None);", 2)
-        self.emit('if (!o)', 1)
+        self.emit('return Py_None;', 2)
-        self.emit('return;', 2)
+        self.emit("}", 1)
         self.emit('', 0)
 
-    def func_end(self):
+    def func_end(self, has_seq):
+        self.emit("return result;", 1)
+        self.emit("failed:", 0)
+        self.emit("Py_XDECREF(value);", 1)
+        self.emit("Py_XDECREF(result);", 1)
+        self.emit("return NULL;", 1)
         self.emit("}", 0)
         self.emit("", 0)
 
     def visitSum(self, sum, name):
-        has_seq = has_sequence(sum.types, True)
+        if is_simple(sum):
+            self.simpleSum(sum, name)
+            return
+        has_seq = has_sequence(sum.types, False)
         self.func_begin(name, has_seq)
-        if not is_simple(sum):
+        self.emit("switch (o->kind) {", 1)
-            self.emit("switch (o->kind) {", 1)
+        for i in range(len(sum.types)):
-            for i in range(len(sum.types)):
+            t = sum.types[i]
-                t = sum.types[i]
+            self.visitConstructor(t, i + 1, name)
-                self.visitConstructor(t, i + 1, name)
+        self.emit("}", 1)
-            self.emit("}", 1)
+        self.emit("", 0)
-            self.emit("", 0)
+        self.func_end(has_seq)
-            self.emit("free(o);", 1)
+        
-        self.func_end()
+    def simpleSum(self, sum, name):
+        self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
+        self.emit("{", 0)
+        self.emit("switch(o) {", 1)
+        for t in sum.types:
+            self.emit("case %s:" % t.name, 2)
+            self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
+            self.emit("return %s_singleton;" % t.name, 3)
+        self.emit("}", 1)
+        self.emit("return NULL; /* cannot happen */", 1)
+        self.emit("}", 0)
 
     def visitProduct(self, prod, name):
-        self.func_begin(name, find_sequence(prod.fields, True))
+        has_seq = find_sequence(prod.fields, False)
+        self.func_begin(name, has_seq)
+        self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
+        self.emit("if (!result) return NULL;", 1)
         for field in prod.fields:
             self.visitField(field, name, 1, True)
         self.emit("", 0)
-        self.emit("free(o);", 1)
+        self.func_end(has_seq)
-        self.func_end()
 
     def visitConstructor(self, cons, enum, name):
         self.emit("case %s_kind:" % cons.name, 1)
+        self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
+        self.emit("if (!result) goto failed;", 2)
         for f in cons.fields:
             self.visitField(f, cons.name, 2, False)
         self.emit("break;", 2)
@@ -453,40 +589,36 @@ class FreeVisitor(PickleVisitor):
             value = "o->%s" % field.name
         else:
             value = "o->v.%s.%s" % (name, field.name)
-        if field.seq:
+        self.set(field, value, depth)
-            self.emitSeq(field, value, depth, emit)
+        emit("if (!value) goto failed;", 0)
-
+        emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0)
-        # XXX need to know the simple types in advance, so that we
+        emit("goto failed;", 1)
-        # don't call free_TYPE() for them.
+        emit("Py_DECREF(value);", 0)
-
-        elif field.opt:
-            emit("if (%s) {" % value, 0)
-            self.free(field, value, depth + 1)
-            emit("}", 0)
-        else:
-            self.free(field, value, depth)
 
     def emitSeq(self, field, value, depth, emit):
-        # specialize for freeing sequences of statements and expressions
+        emit("seq = %s;" % value, 0)
-        if str(field.type) in _SPECIALIZED_SEQUENCES:
+        emit("n = asdl_seq_LEN(seq);", 0)
-            c_code = "free_seq_%ss(%s);" % (field.type, value)
+        emit("value = PyList_New(n);", 0)
-            emit(c_code, 0)
+        emit("if (!value) goto failed;", 0)
-        else:
+        emit("for (i = 0; i < n; i++) {", 0)
-            emit("seq = %s;" % value, 0)
+        self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1)
-            emit("n = asdl_seq_LEN(seq);", 0)
+        emit("if (!value1) goto failed;", 1)
-            emit("for (i = 0; i < n; i++)", 0)
+        emit("PyList_SET_ITEM(value, i, value1);", 1)
-            self.free(field, "asdl_seq_GET(seq, i)", depth + 1)
+        emit("value1 = NULL;", 1)
-            emit("asdl_seq_free(seq);", 0)
+        emit("}", 0)
 
-    def free(self, field, value, depth):
+    def set(self, field, value, depth):
-        if str(field.type) in ("identifier", "string", "object"):
+        if field.seq:
-            ctype = get_c_type(field.type)
+            if field.type.value == "cmpop":
-            self.emit("Py_DECREF((%s)%s);" % (ctype, value), depth)
+                # XXX check that this cast is safe, i.e. works independent on whether
-        elif str(field.type) == "bool":
+                # sizeof(cmpop_ty) != sizeof(void*)
-            return
+                cast = "(PyObject*(*)(void*))"
+            else:
+                cast = ""
+            self.emit("value = ast2obj_list(%s, %sast2obj_%s);" % (value, cast, field.type), depth)
         else:
             ctype = get_c_type(field.type)
-            self.emit("free_%s((%s)%s);" % (field.type, ctype, value), depth)
+            self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
 
 
 class MarshalUtilVisitor(StaticVisitor):
@@ -633,6 +765,16 @@ class MarshalFunctionVisitor(PickleVisitor):
         else:
             emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 0)
             
+class PartingShots(StaticVisitor):
+
+    CODE = """
+PyObject* PyAST_mod2obj(mod_ty t)
+{
+    init_types();
+    return ast2obj_mod(t);
+}
+"""
+
 class ChainOfVisitors:
     def __init__(self, *visitors):
         self.visitors = visitors
@@ -663,6 +805,7 @@ def main(srcfile):
 ##                        FreePrototypeVisitor(f),
                         )
     c.visit(mod)
+    print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
     f.close()
 
     if SRC_DIR:
@@ -674,12 +817,16 @@ def main(srcfile):
     print >> f, '#include "Python.h"'
     print >> f, '#include "%s-ast.h"' % mod.name
     print >> f
-    v = ChainOfVisitors(MarshalPrototypeVisitor(f),
+    v = ChainOfVisitors(
+                        # MarshalPrototypeVisitor(f),
+                        PyTypesDeclareVisitor(f),
+                        PyTypesVisitor(f),
                         FunctionVisitor(f),
 ##                        FreeUtilVisitor(f),
-##                        FreeVisitor(f),
+                        ObjVisitor(f),
-                        MarshalUtilVisitor(f),
+                        #MarshalUtilVisitor(f),
-                        MarshalFunctionVisitor(f),
+                        #MarshalFunctionVisitor(f),
+                        PartingShots(f),
                         )
     v.visit(mod)
     f.close()

Python/bltinmodule.c

@@ -443,7 +443,7 @@ builtin_compile(PyObject *self, PyObject *args)
 	}
 
 	if (supplied_flags &
-	    ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT))
+	    ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST))
 	{
 		PyErr_SetString(PyExc_ValueError,
 				"compile(): unrecognised flags");

Python/pythonrun.c

@@ -1254,6 +1254,11 @@ Py_CompileStringFlags(const char *str, const char *filename, int start,
                 PyArena_Free(arena);
 		return NULL;
         }
+	if (flags->cf_flags & PyCF_ONLY_AST) {
+		PyObject *result = PyAST_mod2obj(mod);
+		PyArena_Free(arena);
+		return result;
+	}
 	co = PyAST_Compile(mod, filename, flags, arena);
         PyArena_Free(arena);
 	return (PyObject *)co;