Merge ast-branch to head

This change implements a new bytecode compiler, based on a
transformation of the parse tree to an abstract syntax defined in
Parser/Python.asdl.

The compiler implementation is not complete, but it is in stable
enough shape to run the entire test suite excepting two disabled
tests.

Include/Python-ast.h

@@ -0,0 +1,418 @@
+/* File automatically generated by ../Parser/asdl_c.py */
+
+#include "asdl.h"
+
+typedef struct _mod *mod_ty;
+
+typedef struct _stmt *stmt_ty;
+
+typedef struct _expr *expr_ty;
+
+typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5,
+                             Param=6 } expr_context_ty;
+
+typedef struct _slice *slice_ty;
+
+typedef enum _boolop { And=1, Or=2 } boolop_ty;
+
+typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7,
+                         RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 }
+                         operator_ty;
+
+typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty;
+
+typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8,
+                      In=9, NotIn=10 } cmpop_ty;
+
+typedef struct _comprehension *comprehension_ty;
+
+typedef struct _excepthandler *excepthandler_ty;
+
+typedef struct _arguments *arguments_ty;
+
+typedef struct _keyword *keyword_ty;
+
+typedef struct _alias *alias_ty;
+
+struct _mod {
+        enum { Module_kind=1, Interactive_kind=2, Expression_kind=3,
+               Suite_kind=4 } kind;
+        union {
+                struct {
+                        asdl_seq *body;
+                } Module;
+                
+                struct {
+                        asdl_seq *body;
+                } Interactive;
+                
+                struct {
+                        expr_ty body;
+                } Expression;
+                
+                struct {
+                        asdl_seq *body;
+                } Suite;
+                
+        } v;
+};
+
+struct _stmt {
+        enum { FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
+               Delete_kind=4, Assign_kind=5, AugAssign_kind=6, Print_kind=7,
+               For_kind=8, While_kind=9, If_kind=10, Raise_kind=11,
+               TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
+               Import_kind=15, ImportFrom_kind=16, Exec_kind=17,
+               Global_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
+               Continue_kind=22 } kind;
+        union {
+                struct {
+                        identifier name;
+                        arguments_ty args;
+                        asdl_seq *body;
+                        asdl_seq *decorators;
+                } FunctionDef;
+                
+                struct {
+                        identifier name;
+                        asdl_seq *bases;
+                        asdl_seq *body;
+                } ClassDef;
+                
+                struct {
+                        expr_ty value;
+                } Return;
+                
+                struct {
+                        asdl_seq *targets;
+                } Delete;
+                
+                struct {
+                        asdl_seq *targets;
+                        expr_ty value;
+                } Assign;
+                
+                struct {
+                        expr_ty target;
+                        operator_ty op;
+                        expr_ty value;
+                } AugAssign;
+                
+                struct {
+                        expr_ty dest;
+                        asdl_seq *values;
+                        bool nl;
+                } Print;
+                
+                struct {
+                        expr_ty target;
+                        expr_ty iter;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } For;
+                
+                struct {
+                        expr_ty test;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } While;
+                
+                struct {
+                        expr_ty test;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } If;
+                
+                struct {
+                        expr_ty type;
+                        expr_ty inst;
+                        expr_ty tback;
+                } Raise;
+                
+                struct {
+                        asdl_seq *body;
+                        asdl_seq *handlers;
+                        asdl_seq *orelse;
+                } TryExcept;
+                
+                struct {
+                        asdl_seq *body;
+                        asdl_seq *finalbody;
+                } TryFinally;
+                
+                struct {
+                        expr_ty test;
+                        expr_ty msg;
+                } Assert;
+                
+                struct {
+                        asdl_seq *names;
+                } Import;
+                
+                struct {
+                        identifier module;
+                        asdl_seq *names;
+                } ImportFrom;
+                
+                struct {
+                        expr_ty body;
+                        expr_ty globals;
+                        expr_ty locals;
+                } Exec;
+                
+                struct {
+                        asdl_seq *names;
+                } Global;
+                
+                struct {
+                        expr_ty value;
+                } Expr;
+                
+        } v;
+        int lineno;
+};
+
+struct _expr {
+        enum { BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
+               Dict_kind=5, ListComp_kind=6, GeneratorExp_kind=7, Yield_kind=8,
+               Compare_kind=9, Call_kind=10, Repr_kind=11, Num_kind=12,
+               Str_kind=13, Attribute_kind=14, Subscript_kind=15, Name_kind=16,
+               List_kind=17, Tuple_kind=18 } kind;
+        union {
+                struct {
+                        boolop_ty op;
+                        asdl_seq *values;
+                } BoolOp;
+                
+                struct {
+                        expr_ty left;
+                        operator_ty op;
+                        expr_ty right;
+                } BinOp;
+                
+                struct {
+                        unaryop_ty op;
+                        expr_ty operand;
+                } UnaryOp;
+                
+                struct {
+                        arguments_ty args;
+                        expr_ty body;
+                } Lambda;
+                
+                struct {
+                        asdl_seq *keys;
+                        asdl_seq *values;
+                } Dict;
+                
+                struct {
+                        expr_ty elt;
+                        asdl_seq *generators;
+                } ListComp;
+                
+                struct {
+                        expr_ty elt;
+                        asdl_seq *generators;
+                } GeneratorExp;
+                
+                struct {
+                        expr_ty value;
+                } Yield;
+                
+                struct {
+                        expr_ty left;
+                        asdl_seq *ops;
+                        asdl_seq *comparators;
+                } Compare;
+                
+                struct {
+                        expr_ty func;
+                        asdl_seq *args;
+                        asdl_seq *keywords;
+                        expr_ty starargs;
+                        expr_ty kwargs;
+                } Call;
+                
+                struct {
+                        expr_ty value;
+                } Repr;
+                
+                struct {
+                        object n;
+                } Num;
+                
+                struct {
+                        string s;
+                } Str;
+                
+                struct {
+                        expr_ty value;
+                        identifier attr;
+                        expr_context_ty ctx;
+                } Attribute;
+                
+                struct {
+                        expr_ty value;
+                        slice_ty slice;
+                        expr_context_ty ctx;
+                } Subscript;
+                
+                struct {
+                        identifier id;
+                        expr_context_ty ctx;
+                } Name;
+                
+                struct {
+                        asdl_seq *elts;
+                        expr_context_ty ctx;
+                } List;
+                
+                struct {
+                        asdl_seq *elts;
+                        expr_context_ty ctx;
+                } Tuple;
+                
+        } v;
+        int lineno;
+};
+
+struct _slice {
+        enum { Ellipsis_kind=1, Slice_kind=2, ExtSlice_kind=3, Index_kind=4 }
+               kind;
+        union {
+                struct {
+                        expr_ty lower;
+                        expr_ty upper;
+                        expr_ty step;
+                } Slice;
+                
+                struct {
+                        asdl_seq *dims;
+                } ExtSlice;
+                
+                struct {
+                        expr_ty value;
+                } Index;
+                
+        } v;
+};
+
+struct _comprehension {
+        expr_ty target;
+        expr_ty iter;
+        asdl_seq *ifs;
+};
+
+struct _excepthandler {
+        expr_ty type;
+        expr_ty name;
+        asdl_seq *body;
+};
+
+struct _arguments {
+        asdl_seq *args;
+        identifier vararg;
+        identifier kwarg;
+        asdl_seq *defaults;
+};
+
+struct _keyword {
+        identifier arg;
+        expr_ty value;
+};
+
+struct _alias {
+        identifier name;
+        identifier asname;
+};
+
+mod_ty Module(asdl_seq * body);
+mod_ty Interactive(asdl_seq * body);
+mod_ty Expression(expr_ty body);
+mod_ty Suite(asdl_seq * body);
+stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
+                    asdl_seq * decorators, int lineno);
+stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
+                 lineno);
+stmt_ty Return(expr_ty value, int lineno);
+stmt_ty Delete(asdl_seq * targets, int lineno);
+stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno);
+stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno);
+stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno);
+stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
+            int lineno);
+stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
+stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
+stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno);
+stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
+                  lineno);
+stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno);
+stmt_ty Assert(expr_ty test, expr_ty msg, int lineno);
+stmt_ty Import(asdl_seq * names, int lineno);
+stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno);
+stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno);
+stmt_ty Global(asdl_seq * names, int lineno);
+stmt_ty Expr(expr_ty value, int lineno);
+stmt_ty Pass(int lineno);
+stmt_ty Break(int lineno);
+stmt_ty Continue(int lineno);
+expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno);
+expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno);
+expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno);
+expr_ty Lambda(arguments_ty args, expr_ty body, int lineno);
+expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno);
+expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno);
+expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno);
+expr_ty Yield(expr_ty value, int lineno);
+expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
+                lineno);
+expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
+             starargs, expr_ty kwargs, int lineno);
+expr_ty Repr(expr_ty value, int lineno);
+expr_ty Num(object n, int lineno);
+expr_ty Str(string s, int lineno);
+expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
+                  lineno);
+expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
+                  lineno);
+expr_ty Name(identifier id, expr_context_ty ctx, int lineno);
+expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno);
+expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno);
+slice_ty Ellipsis(void);
+slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step);
+slice_ty ExtSlice(asdl_seq * dims);
+slice_ty Index(expr_ty value);
+comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs);
+excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body);
+arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
+                       asdl_seq * defaults);
+keyword_ty keyword(identifier arg, expr_ty value);
+alias_ty alias(identifier name, identifier asname);
+void free_mod(mod_ty);
+void free_stmt(stmt_ty);
+void free_expr(expr_ty);
+void free_expr_context(expr_context_ty);
+void free_slice(slice_ty);
+void free_boolop(boolop_ty);
+void free_operator(operator_ty);
+void free_unaryop(unaryop_ty);
+void free_cmpop(cmpop_ty);
+void free_comprehension(comprehension_ty);
+void free_excepthandler(excepthandler_ty);
+void free_arguments(arguments_ty);
+void free_keyword(keyword_ty);
+void free_alias(alias_ty);
+int marshal_write_mod(PyObject **, int *, mod_ty);
+int marshal_write_stmt(PyObject **, int *, stmt_ty);
+int marshal_write_expr(PyObject **, int *, expr_ty);
+int marshal_write_expr_context(PyObject **, int *, expr_context_ty);
+int marshal_write_slice(PyObject **, int *, slice_ty);
+int marshal_write_boolop(PyObject **, int *, boolop_ty);
+int marshal_write_operator(PyObject **, int *, operator_ty);
+int marshal_write_unaryop(PyObject **, int *, unaryop_ty);
+int marshal_write_cmpop(PyObject **, int *, cmpop_ty);
+int marshal_write_comprehension(PyObject **, int *, comprehension_ty);
+int marshal_write_excepthandler(PyObject **, int *, excepthandler_ty);
+int marshal_write_arguments(PyObject **, int *, arguments_ty);
+int marshal_write_keyword(PyObject **, int *, keyword_ty);
+int marshal_write_alias(PyObject **, int *, alias_ty);

Include/Python.h

@@ -128,8 +128,7 @@
 #include "pystrtod.h"
 
 /* _Py_Mangle is defined in compile.c */
-PyAPI_FUNC(int) _Py_Mangle(char *p, char *name, \
+PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
-				 char *buffer, size_t maxlen);
 
 /* PyArg_GetInt is deprecated and should not be used, use PyArg_Parse(). */
 #define PyArg_GetInt(v, a)	PyArg_Parse((v), "i", (a))

Include/asdl.h

@@ -0,0 +1,54 @@
+#ifndef Py_ASDL_H
+#define Py_ASDL_H
+
+typedef PyObject * identifier;
+typedef PyObject * string;
+typedef PyObject * object;
+
+typedef enum {false, true} bool;
+
+/* It would be nice if the code generated by asdl_c.py was completely
+   independent of Python, but it is a goal the requires too much work
+   at this stage.  So, for example, I'll represent identifiers as
+   interned Python strings.
+*/
+
+/* XXX A sequence should be typed so that its use can be typechecked. */
+
+/* XXX We shouldn't pay for offset when we don't need APPEND. */
+
+typedef struct {
+    int size;
+    int offset;
+    void *elements[1];
+} asdl_seq;
+
+asdl_seq *asdl_seq_new(int size);
+void asdl_seq_free(asdl_seq *);
+
+#ifdef Py_DEBUG
+#define asdl_seq_GET(S, I) (S)->elements[(I)]
+#define asdl_seq_SET(S, I, V) { \
+        int _asdl_i = (I); \
+        assert((S) && _asdl_i < (S)->size); \
+        (S)->elements[_asdl_i] = (V); \
+}
+#define asdl_seq_APPEND(S, V) { \
+        assert((S) && (S)->offset < (S)->size); \
+        (S)->elements[(S)->offset++] = (V); \
+}
+#else
+#define asdl_seq_GET(S, I) (S)->elements[(I)]
+#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
+#define asdl_seq_APPEND(S, V) (S)->elements[(S)->offset++] = (V)
+#endif
+#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
+
+/* Routines to marshal the basic types. */
+int marshal_write_int(PyObject **, int *, int);
+int marshal_write_bool(PyObject **, int *, bool);
+int marshal_write_identifier(PyObject **, int *, identifier);
+int marshal_write_string(PyObject **, int *, string);
+int marshal_write_object(PyObject **, int *, object);
+
+#endif /* !Py_ASDL_H */

Include/ast.h

@@ -0,0 +1,13 @@
+#ifndef Py_AST_H
+#define Py_AST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern DL_IMPORT(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
+					const char *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_AST_H */

Include/code.h

@@ -0,0 +1,73 @@
+/* Definitions for bytecode */
+
+#ifndef Py_CODE_H
+#define Py_CODE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Bytecode object */
+typedef struct {
+    PyObject_HEAD
+    int co_argcount;		/* #arguments, except *args */
+    int co_nlocals;		/* #local variables */
+    int co_stacksize;		/* #entries needed for evaluation stack */
+    int co_flags;		/* CO_..., see below */
+    PyObject *co_code;		/* instruction opcodes */
+    PyObject *co_consts;	/* list (constants used) */
+    PyObject *co_names;		/* list of strings (names used) */
+    PyObject *co_varnames;	/* tuple of strings (local variable names) */
+    PyObject *co_freevars;	/* tuple of strings (free variable names) */
+    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
+    /* The rest doesn't count for hash/cmp */
+    PyObject *co_filename;	/* string (where it was loaded from) */
+    PyObject *co_name;		/* string (name, for reference) */
+    int co_firstlineno;		/* first source line number */
+    PyObject *co_lnotab;	/* string (encoding addr<->lineno mapping) */
+} PyCodeObject;
+
+/* Masks for co_flags above */
+#define CO_OPTIMIZED	0x0001
+#define CO_NEWLOCALS	0x0002
+#define CO_VARARGS	0x0004
+#define CO_VARKEYWORDS	0x0008
+#define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
+/* The CO_NOFREE flag is set if there are no free or cell variables.
+   This information is redundant, but it allows a single flag test
+   to determine whether there is any extra work to be done when the
+   call frame it setup.
+*/
+#define CO_NOFREE       0x0040
+/* XXX Temporary hack.  Until generators are a permanent part of the
+   language, we need a way for a code object to record that generators
+   were *possible* when it was compiled.  This is so code dynamically
+   compiled *by* a code object knows whether to allow yield stmts.  In
+   effect, this passes on the "from __future__ import generators" state
+   in effect when the code block was compiled. */
+#define CO_GENERATOR_ALLOWED    0x1000 /* no longer used in an essential way */
+#define CO_FUTURE_DIVISION    	0x2000
+
+#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
+
+extern DL_IMPORT(PyTypeObject) PyCode_Type;
+
+#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
+#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
+
+/* Public interface */
+DL_IMPORT(PyCodeObject *) PyCode_New(
+	int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
+	PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *); 
+        /* same as struct above */
+DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
+
+/* for internal use only */
+#define _PyCode_GETCODEPTR(co, pp) \
+	((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \
+	 ((co)->co_code, 0, (void **)(pp)))
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CODE_H */

Include/compile.h

@@ -1,5 +1,6 @@
-
+#ifndef Py_CODE_H
-/* Definitions for bytecode */
+#include "code.h"
+#endif
 
 #ifndef Py_COMPILE_H
 #define Py_COMPILE_H
@@ -7,55 +8,6 @@
 extern "C" {
 #endif
 
-/* Bytecode object */
-typedef struct {
-    PyObject_HEAD
-    int co_argcount;		/* #arguments, except *args */
-    int co_nlocals;		/* #local variables */
-    int co_stacksize;		/* #entries needed for evaluation stack */
-    int co_flags;		/* CO_..., see below */
-    PyObject *co_code;		/* instruction opcodes */
-    PyObject *co_consts;	/* list (constants used) */
-    PyObject *co_names;		/* list of strings (names used) */
-    PyObject *co_varnames;	/* tuple of strings (local variable names) */
-    PyObject *co_freevars;	/* tuple of strings (free variable names) */
-    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
-    /* The rest doesn't count for hash/cmp */
-    PyObject *co_filename;	/* string (where it was loaded from) */
-    PyObject *co_name;		/* string (name, for reference) */
-    int co_firstlineno;		/* first source line number */
-    PyObject *co_lnotab;	/* string (encoding addr<->lineno mapping) */
-} PyCodeObject;
-
-/* Masks for co_flags above */
-#define CO_OPTIMIZED	0x0001
-#define CO_NEWLOCALS	0x0002
-#define CO_VARARGS	0x0004
-#define CO_VARKEYWORDS	0x0008
-#define CO_NESTED       0x0010
-#define CO_GENERATOR    0x0020
-/* The CO_NOFREE flag is set if there are no free or cell variables.
-   This information is redundant, but it allows a single flag test
-   to determine whether there is any extra work to be done when the
-   call frame it setup.
-*/
-#define CO_NOFREE       0x0040
-/* XXX Temporary hack.  Until generators are a permanent part of the
-   language, we need a way for a code object to record that generators
-   were *possible* when it was compiled.  This is so code dynamically
-   compiled *by* a code object knows whether to allow yield stmts.  In
-   effect, this passes on the "from __future__ import generators" state
-   in effect when the code block was compiled. */
-#define CO_GENERATOR_ALLOWED    0x1000 /* no longer used in an essential way */
-#define CO_FUTURE_DIVISION    	0x2000
-
-PyAPI_DATA(PyTypeObject) PyCode_Type;
-
-#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
-#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
-
-#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
-
 /* Public interface */
 struct _node; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
@@ -68,19 +20,22 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
 /* Future feature support */
 
 typedef struct {
-    int ff_found_docstring;
+    int ff_features;      /* flags set by future statements */
-    int ff_last_lineno;
+    int ff_lineno;        /* line number of last future statement */
-    int ff_features;
 } PyFutureFeatures;
 
-PyAPI_FUNC(PyFutureFeatures *) PyNode_Future(struct _node *, const char *);
-PyAPI_FUNC(PyCodeObject *) PyNode_CompileFlags(struct _node *, const char *,
-					      PyCompilerFlags *);
-
 #define FUTURE_NESTED_SCOPES "nested_scopes"
 #define FUTURE_GENERATORS "generators"
 #define FUTURE_DIVISION "division"
 
+struct _mod; /* Declare the existence of this type */
+DL_IMPORT(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
+					PyCompilerFlags *);
+DL_IMPORT(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+
+#define ERR_LATE_FUTURE \
+"from __future__ imports must occur at the beginning of the file"
+
 #ifdef __cplusplus
 }
 #endif

Include/pyport.h

@@ -583,6 +583,7 @@ typedef	struct fd_set {
 
 #ifndef INT_MAX
 #define INT_MAX 2147483647
+#define INT_MIN (-INT_MAX - 1)
 #endif
 
 #ifndef LONG_MAX

Include/pythonrun.h

@@ -29,46 +29,37 @@ PyAPI_FUNC(int) Py_IsInitialized(void);
 PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
 PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 
-PyAPI_FUNC(int) PyRun_AnyFile(FILE *, const char *);
+PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *, const char *, int);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, char *, int, PyCompilerFlags *);
-
-PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-
-PyAPI_FUNC(int) PyRun_SimpleString(const char *);
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFile(FILE *, const char *);
-PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *, const char *, int);
 PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *, const char *);
 PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *, const char *);
 PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
 
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseString(const char *, int);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, 
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseFile(FILE *, const char *, int);
+						 int, PyCompilerFlags *flags);
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int, 
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *,
+					       char *, char *,
-								  const char *,
+                                               PyCompilerFlags *, int *);
-								  int,
+#define PyParser_SimpleParseString(S, B) \
-								  int);
+        PyParser_SimpleParseStringFlags(S, B, 0)
+#define PyParser_SimpleParseFile(FP, S, B) \
+        PyParser_SimpleParseFileFlags(FP, S, B, 0)
+PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, 
+							  int);
 PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
 							int, int);
 
-PyAPI_FUNC(PyObject *) PyRun_String(const char *, int, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, 
-PyAPI_FUNC(PyObject *) PyRun_File(FILE *, const char *, int, PyObject *, PyObject *);
+					 PyObject *, PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *, const char *, int,
-				   PyObject *, PyObject *, int);
-PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *,
-					PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *, const char *, int, PyObject *, 
-				      PyObject *, PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, PyObject *, 
-					PyObject *, int, PyCompilerFlags *);
 
-PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, 
+					 PyObject *, PyObject *, int, 
+					 PyCompilerFlags *);
+
+#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL)
 PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int,
-					    PyCompilerFlags *);
+					     PyCompilerFlags *);
 PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
 
 PyAPI_FUNC(void) PyErr_Print(void);
@@ -84,6 +75,25 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
 /* Bootstrap */
 PyAPI_FUNC(int) Py_Main(int argc, char **argv);
 
+/* Use macros for a bunch of old variants */
+#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
+#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL)
+#define PyRun_AnyFileEx(fp, name, closeit) \
+	PyRun_AnyFileExFlags(fp, name, closeit, NULL)
+#define PyRun_AnyFileFlags(fp, name, flags) \
+	PyRun_AnyFileExFlags(fp, name, 0, flags)
+#define PyRun_SimpleString(s, f) PyRunSimpleStringFlags(s, f, NULL)
+#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL)
+#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL)
+#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL)
+#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL)
+#define PyRun_File(fp, p, s, g, l) \
+        PyRun_FileExFlags(fp, p, s, g, l, 0, NULL)
+#define PyRun_FileEx(fp, p, s, g, l, c) \
+        PyRun_FileExFlags(fp, p, s, g, l, c, NULL)
+#define PyRun_FileFlags(fp, p, s, g, l, flags) \
+        PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
+
 /* In getpath.c */
 PyAPI_FUNC(char *) Py_GetProgramFullPath(void);
 PyAPI_FUNC(char *) Py_GetPrefix(void);

Include/symtable.h

@@ -4,65 +4,60 @@
 extern "C" {
 #endif
 
-/* A symbol table is constructed each time PyNode_Compile() is
+typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
-   called.  The table walks the entire parse tree and identifies each
+    block_ty;
-   use or definition of a variable. 
-
-   The symbol table contains a dictionary for each code block in a
-   module: The symbol dictionary for the block.  They keys of these
-   dictionaries are the name of all variables used or defined in the
-   block; the integer values are used to store several flags,
-   e.g. DEF_PARAM indicates that a variable is a parameter to a
-   function. 
-*/
 
 struct _symtable_entry;
 
 struct symtable {
-	int st_pass;             /* pass == 1 or 2 */
 	const char *st_filename; /* name of file being compiled */
 	struct _symtable_entry *st_cur; /* current symbol table entry */
+	struct _symtable_entry *st_top; /* module entry */
 	PyObject *st_symbols;    /* dictionary of symbol table entries */
         PyObject *st_stack;      /* stack of namespace info */
 	PyObject *st_global;     /* borrowed ref to MODULE in st_symbols */
-	int st_nscopes;          /* number of scopes */
+	int st_nblocks;          /* number of blocks */
-	int st_errors;           /* number of errors */
 	char *st_private;        /* name of current class or NULL */
+        int st_tmpname;          /* temporary name counter */
 	PyFutureFeatures *st_future; /* module's future features */
 };
 
 typedef struct _symtable_entry {
 	PyObject_HEAD
-	PyObject *ste_id;        /* int: key in st_symbols) */
+	PyObject *ste_id;        /* int: key in st_symbols */
-	PyObject *ste_symbols;   /* dict: name to flags) */
+	PyObject *ste_symbols;   /* dict: name to flags */
-	PyObject *ste_name;      /* string: name of scope */
+	PyObject *ste_name;      /* string: name of block */
 	PyObject *ste_varnames;  /* list of variable names */
 	PyObject *ste_children;  /* list of child ids */
-	int ste_type;            /* module, class, or function */
+	block_ty ste_type;       /* module, class, or function */
-	int ste_lineno;          /* first line of scope */
+	int ste_unoptimized;     /* false if namespace is optimized */
-	int ste_optimized;       /* true if namespace can't be optimized */
+	int ste_nested : 1;      /* true if block is nested */
-	int ste_nested;          /* true if scope is nested */
+	int ste_free : 1;        /* true if block has free variables */
-	int ste_child_free;      /* true if a child scope has free variables,
+	int ste_child_free : 1;  /* true if a child block has free variables,
 				    including free refs to globals */
-	int ste_generator;       /* true if namespace is a generator */
+	int ste_generator : 1;   /* true if namespace is a generator */
+	int ste_varargs : 1;     /* true if block has varargs */
+	int ste_varkeywords : 1; /* true if block has varkeywords */
+	int ste_lineno;          /* first line of block */
 	int ste_opt_lineno;      /* lineno of last exec or import * */
-	int ste_tmpname;         /* temporary name counter */
+	int ste_tmpname;         /* counter for listcomp temp vars */
 	struct symtable *ste_table;
-} PySymtableEntryObject;
+} PySTEntryObject;
 
-PyAPI_DATA(PyTypeObject) PySymtableEntry_Type;
+PyAPI_DATA(PyTypeObject) PySTEntry_Type;
 
-#define PySymtableEntry_Check(op) ((op)->ob_type == &PySymtableEntry_Type)
+#define PySTEntry_Check(op) ((op)->ob_type == &PySTEntry_Type)
 
-PyAPI_FUNC(PyObject *) PySymtableEntry_New(struct symtable *,
+PyAPI_FUNC(PySTEntryObject *) \
-						 char *, int, int);
+	PySTEntry_New(struct symtable *, identifier, block_ty, void *, int);
+PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
+
+PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, 
+					      PyFutureFeatures *);
+PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
-PyAPI_FUNC(struct symtable *) PyNode_CompileSymtable(struct _node *, const char *);
 PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 
-
-#define TOP "global"
-
 /* Flags for def-use information */
 
 #define DEF_GLOBAL 1           /* global stmt */
@@ -72,16 +67,19 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_STAR 2<<3          /* parameter is star arg */
 #define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
 #define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
-#define DEF_FREE 2<<6          /* name used but not defined in nested scope */
+#define DEF_FREE 2<<6          /* name used but not defined in nested block */
 #define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
 #define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
 #define DEF_IMPORT 2<<9        /* assignment occurred via import */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 
-#define TYPE_FUNCTION 1
+/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
-#define TYPE_CLASS 2
+   table.  GLOBAL is returned from PyST_GetScope() for either of them. 
-#define TYPE_MODULE 3
+   It is stored in ste_symbols at bits 12-14.
+*/
+#define SCOPE_OFF 11
+#define SCOPE_MASK 7
 
 #define LOCAL 1
 #define GLOBAL_EXPLICIT 2
@@ -89,9 +87,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define FREE 4
 #define CELL 5
 
+/* The following three names are used for the ste_unoptimized bit field */
 #define OPT_IMPORT_STAR 1
 #define OPT_EXEC 2
 #define OPT_BARE_EXEC 4
+#define OPT_TOPLEVEL 8  /* top-level names, including eval and exec */
+
+#define GENERATOR 1
+#define GENERATOR_EXPRESSION 2
 
 #define GENERATOR 1
 #define GENERATOR_EXPRESSION 2

Lib/cgitb.py

@@ -22,6 +22,7 @@ The default handler displays output as HTML.
 """
 
 __author__ = 'Ka-Ping Yee'
+
 __version__ = '$Revision$'
 
 import sys

Lib/compiler/pyassem.py

@@ -364,16 +364,15 @@ class PyFlowGraph(FlowGraph):
 
     def getCode(self):
         """Get a Python code object"""
-        if self.stage == RAW:
+        assert self.stage == RAW
-            self.computeStackDepth()
+        self.computeStackDepth()
-            self.flattenGraph()
+        self.flattenGraph()
-        if self.stage == FLAT:
+        assert self.stage == FLAT
-            self.convertArgs()
+        self.convertArgs()
-        if self.stage == CONV:
+        assert self.stage == CONV
-            self.makeByteCode()
+        self.makeByteCode()
-        if self.stage == DONE:
+        assert self.stage == DONE
-            return self.newCodeObject()
+        return self.newCodeObject()
-        raise RuntimeError, "inconsistent PyFlowGraph state"
 
     def dump(self, io=None):
         if io:

Lib/pydoc.py

@@ -36,6 +36,7 @@ Reference Manual pages.
 
 __author__ = "Ka-Ping Yee <ping@lfw.org>"
 __date__ = "26 February 2001"
+
 __version__ = "$Revision$"
 __credits__ = """Guido van Rossum, for an excellent programming language.
 Tommy Burnette, the original creator of manpy.

Lib/test/output/test_grammar

@@ -34,6 +34,7 @@ continue + try/except ok
 continue + try/finally ok
 testing continue and break in try/except in loop
 return_stmt
+yield_stmt
 raise_stmt
 import_name
 import_from

Lib/test/output/test_profile

@@ -7,7 +7,7 @@ test_profile
        12    0.000    0.000    0.012    0.001 :0(hasattr)
         8    0.000    0.000    0.000    0.000 :0(range)
         1    0.000    0.000    0.000    0.000 :0(setprofile)
-        1    0.000    0.000    1.000    1.000 <string>:1(?)
+        1    0.000    0.000    1.000    1.000 <string>:1(<module>)
         0    0.000             0.000          profile:0(profiler)
         1    0.000    0.000    1.000    1.000 profile:0(testfunc())
         1    0.400    0.400    1.000    1.000 test_profile.py:23(testfunc)

Lib/test/test_code.py

@@ -0,0 +1,85 @@
+"""This module includes tests of the code object representation.
+
+>>> def f(x):
+...     def g(y):
+...         return x + y
+...     return g
+...
+
+>>> dump(f.func_code)
+name: f
+argcount: 1
+names: ()
+varnames: ('x', 'g')
+cellvars: ('x',)
+freevars: ()
+nlocals: 2
+flags: 3
+consts: ('None', '<code object g>')
+
+>>> dump(f(4).func_code)
+name: g
+argcount: 1
+names: ()
+varnames: ('y',)
+cellvars: ()
+freevars: ('x',)
+nlocals: 1
+flags: 19
+consts: ('None',)
+
+>>> def h(x, y):
+...     a = x + y
+...     b = x - y
+...     c = a * b
+...     return c
+... 
+>>> dump(h.func_code)
+name: h
+argcount: 2
+names: ()
+varnames: ('x', 'y', 'a', 'b', 'c')
+cellvars: ()
+freevars: ()
+nlocals: 5
+flags: 67
+consts: ('None',)
+
+>>> def attrs(obj):
+...     print obj.attr1
+...     print obj.attr2
+...     print obj.attr3
+
+>>> dump(attrs.func_code)
+name: attrs
+argcount: 1
+names: ('attr1', 'attr2', 'attr3')
+varnames: ('obj',)
+cellvars: ()
+freevars: ()
+nlocals: 1
+flags: 67
+consts: ('None',)
+
+"""
+
+def consts(t):
+    """Yield a doctest-safe sequence of object reprs."""
+    for elt in t:
+        r = repr(elt)
+        if r.startswith("<code object"):
+            yield "<code object %s>" % elt.co_name
+        else:
+            yield r
+
+def dump(co):
+    """Print out a text representation of a code object."""
+    for attr in ["name", "argcount", "names", "varnames", "cellvars",
+                 "freevars", "nlocals", "flags"]:
+        print "%s: %s" % (attr, getattr(co, "co_" + attr))
+    print "consts:", tuple(consts(co.co_consts))
+
+def test_main(verbose=None):
+    from test.test_support import run_doctest
+    from test import test_code
+    run_doctest(test_code, verbose)

Lib/test/test_doctest.py

@@ -1559,11 +1559,11 @@ Run the debugger on the docstring, and then restore sys.stdin.
 
     >>> try: doctest.debug_src(s)
     ... finally: sys.stdin = real_stdin
-    > <string>(1)?()
+    > <string>(1)<module>()
     (Pdb) next
     12
     --Return--
-    > <string>(1)?()->None
+    > <string>(1)<module>()->None
     (Pdb) print x
     12
     (Pdb) continue
@@ -1601,7 +1601,7 @@ def test_pdb_set_trace():
       >>> try: runner.run(test)
       ... finally: sys.stdin = real_stdin
       --Return--
-      > <doctest foo[1]>(1)?()->None
+      > <doctest foo[1]>(1)<module>()->None
       -> import pdb; pdb.set_trace()
       (Pdb) print x
       42
@@ -1637,7 +1637,7 @@ def test_pdb_set_trace():
       (Pdb) print y
       2
       (Pdb) up
-      > <doctest foo[1]>(1)?()
+      > <doctest foo[1]>(1)<module>()
       -> calls_set_trace()
       (Pdb) print x
       1
@@ -1686,7 +1686,7 @@ def test_pdb_set_trace():
       [EOF]
       (Pdb) next
       --Return--
-      > <doctest foo[2]>(1)?()->None
+      > <doctest foo[2]>(1)<module>()->None
       -> f(3)
       (Pdb) list
         1  -> f(3)
@@ -1779,7 +1779,7 @@ def test_pdb_set_trace_nested():
     (Pdb) print y
     1
     (Pdb) up
-    > <doctest foo[1]>(1)?()
+    > <doctest foo[1]>(1)<module>()
     -> calls_set_trace()
     (Pdb) print foo
     *** NameError: name 'foo' is not defined

Lib/test/test_eof.py

@@ -7,21 +7,21 @@ from test import test_support
 
 class EOFTestCase(unittest.TestCase):
     def test_EOFC(self):
+        expect = "EOL while scanning single-quoted string (<string>, line 1)"
         try:
             eval("""'this is a test\
             """)
         except SyntaxError, msg:
-            self.assertEqual(str(msg),
+            self.assertEqual(str(msg), expect)
-                             "EOL while scanning single-quoted string (line 1)")
         else:
             raise test_support.TestFailed
 
     def test_EOFS(self):
+        expect = "EOF while scanning triple-quoted string (<string>, line 1)"
         try:
             eval("""'''this is a test""")
         except SyntaxError, msg:
-            self.assertEqual(str(msg),
+            self.assertEqual(str(msg), expect)
-                             "EOF while scanning triple-quoted string (line 1)")
         else:
             raise test_support.TestFailed
 

Lib/test/test_generators.py

@@ -774,7 +774,7 @@ These are fine:
 ...         try:
 ...             1//0
 ...         except ZeroDivisionError:
-...             yield 666  # bad because *outer* try has finally
+...             yield 666 
 ...         except:
 ...             pass
 ...     finally:

Lib/test/test_genexps.py

@@ -125,13 +125,12 @@ Verify that syntax error's are raised for genexps used as lvalues
     >>> (y for y in (1,2)) = 10
     Traceback (most recent call last):
        ...
-    SyntaxError: assign to generator expression not possible
+    SyntaxError: assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[38]>, line 1)
 
     >>> (y for y in (1,2)) += 10
     Traceback (most recent call last):
        ...
-    SyntaxError: augmented assign to tuple literal, yield, or generator expression not possible
+    SyntaxError: augmented assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[39]>, line 1)
-
 
 
 ########### Tests borrowed from or inspired by test_generators.py ############

Lib/test/test_grammar.py

@@ -8,7 +8,7 @@
 # regression test, the filterwarnings() call has been added to
 # regrtest.py.
 
-from test.test_support import TestFailed, verify, check_syntax
+from test.test_support import TestFailed, verify, vereq, check_syntax
 import sys
 
 print '1. Parser'
@@ -157,28 +157,31 @@ def f2(one_argument): pass
 def f3(two, arguments): pass
 def f4(two, (compound, (argument, list))): pass
 def f5((compound, first), two): pass
-verify(f2.func_code.co_varnames == ('one_argument',))
+vereq(f2.func_code.co_varnames, ('one_argument',))
-verify(f3.func_code.co_varnames == ('two', 'arguments'))
+vereq(f3.func_code.co_varnames, ('two', 'arguments'))
 if sys.platform.startswith('java'):
-    verify(f4.func_code.co_varnames ==
+    vereq(f4.func_code.co_varnames,
            ('two', '(compound, (argument, list))', 'compound', 'argument',
                         'list',))
-    verify(f5.func_code.co_varnames ==
+    vereq(f5.func_code.co_varnames,
            ('(compound, first)', 'two', 'compound', 'first'))
 else:
-    verify(f4.func_code.co_varnames == ('two', '.2', 'compound',
+    vereq(f4.func_code.co_varnames,
-                                        'argument',  'list'))
+          ('two', '.1', 'compound', 'argument',  'list'))
-    verify(f5.func_code.co_varnames == ('.0', 'two', 'compound', 'first'))
+    vereq(f5.func_code.co_varnames,
+          ('.0', 'two', 'compound', 'first'))
 def a1(one_arg,): pass
 def a2(two, args,): pass
 def v0(*rest): pass
 def v1(a, *rest): pass
 def v2(a, b, *rest): pass
 def v3(a, (b, c), *rest): return a, b, c, rest
+# ceval unpacks the formal arguments into the first argcount names;
+# thus, the names nested inside tuples must appear after these names.
 if sys.platform.startswith('java'):
     verify(v3.func_code.co_varnames == ('a', '(b, c)', 'rest', 'b', 'c'))
 else:
-    verify(v3.func_code.co_varnames == ('a', '.2', 'rest', 'b', 'c'))
+    vereq(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
 verify(v3(1, (2, 3), 4) == (1, 2, 3, (4,)))
 def d01(a=1): pass
 d01()
@@ -410,6 +413,10 @@ def g1(): return
 def g2(): return 1
 g1()
 x = g2()
+check_syntax("class foo:return 1")
+
+print 'yield_stmt'
+check_syntax("class foo:yield 1")
 
 print 'raise_stmt' # 'raise' test [',' test]
 try: raise RuntimeError, 'just testing'

Lib/test/test_import.py

@@ -192,3 +192,16 @@ def test_failing_reload():
             del sys.modules[TESTFN]
 
 test_failing_reload()
+
+def test_import_name_binding():
+    # import x.y.z binds x in the current namespace
+    import test as x
+    import test.test_support
+    assert x is test, x.__name__
+    assert hasattr(test.test_support, "__file__")
+
+    # import x.y.z as w binds z as w
+    import test.test_support as y
+    assert y is test.test_support, y.__name__
+
+test_import_name_binding()

Lib/test/test_parser.py

@@ -411,10 +411,32 @@ class IllegalSyntaxTestCase(unittest.TestCase):
                 (0, ''))
         self.check_bad_tree(tree, "malformed global ast")
 
+
+class CompileTestCase(unittest.TestCase):
+
+    # These tests are very minimal. :-(
+
+    def test_compile_expr(self):
+        st = parser.expr('2 + 3')
+        code = parser.compilest(st)
+        self.assertEquals(eval(code), 5)
+
+    def test_compile_suite(self):
+        st = parser.suite('x = 2; y = x + 3')
+        code = parser.compilest(st)
+        globs = {}
+        exec code in globs
+        self.assertEquals(globs['y'], 5)
+
+    def test_compile_error(self):
+        st = parser.suite('1 = 3 + 4')
+        self.assertRaises(SyntaxError, parser.compilest, st)
+
 def test_main():
     test_support.run_unittest(
         RoundtripLegalSyntaxTestCase,
-        IllegalSyntaxTestCase
+        IllegalSyntaxTestCase,
+        CompileTestCase,
     )
 
 

Lib/test/test_repr.py

@@ -123,7 +123,7 @@ class ReprTests(unittest.TestCase):
 
     def test_lambda(self):
         self.failUnless(repr(lambda x: x).startswith(
-            "<function <lambda"))
+            "<function lambda"))
         # XXX anonymous functions?  see func_repr
 
     def test_builtin_function(self):

Lib/test/test_scope.py

@@ -1,4 +1,4 @@
-from test.test_support import verify, TestFailed, check_syntax
+from test.test_support import verify, TestFailed, check_syntax, vereq
 
 import warnings
 warnings.filterwarnings("ignore", r"import \*", SyntaxWarning, "<string>")
@@ -13,8 +13,8 @@ def make_adder(x):
 inc = make_adder(1)
 plus10 = make_adder(10)
 
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(-2) == 8)
+vereq(plus10(-2), 8)
 
 print "2. extra nesting"
 
@@ -28,8 +28,8 @@ def make_adder2(x):
 inc = make_adder2(1)
 plus10 = make_adder2(10)
 
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(-2) == 8)
+vereq(plus10(-2), 8)
 
 print "3. simple nesting + rebinding"
 
@@ -42,8 +42,8 @@ def make_adder3(x):
 inc = make_adder3(0)
 plus10 = make_adder3(9)
 
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(-2) == 8)
+vereq(plus10(-2), 8)
 
 print "4. nesting with global but no free"
 
@@ -58,10 +58,10 @@ def make_adder4(): # XXX add exta level of indirection
 
 global_x = 1
 adder = make_adder4()
-verify(adder(1) == 2)
+vereq(adder(1), 2)
 
 global_x = 10
-verify(adder(-2) == 8)
+vereq(adder(-2), 8)
 
 print "5. nesting through class"
 
@@ -74,8 +74,8 @@ def make_adder5(x):
 inc = make_adder5(1)
 plus10 = make_adder5(10)
 
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(-2) == 8)
+vereq(plus10(-2), 8)
 
 print "6. nesting plus free ref to global"
 
@@ -89,8 +89,8 @@ def make_adder6(x):
 inc = make_adder6(1)
 plus10 = make_adder6(10)
 
-verify(inc(1) == 11) # there's only one global
+vereq(inc(1), 11) # there's only one global
-verify(plus10(-2) == 8)
+vereq(plus10(-2), 8)
 
 print "7. nearest enclosing scope"
 
@@ -103,7 +103,7 @@ def f(x):
     return g(2)
 
 test_func = f(10)
-verify(test_func(5) == 47)
+vereq(test_func(5), 47)
 
 print "8. mixed freevars and cellvars"
 
@@ -123,7 +123,7 @@ def f(x, y, z):
 
 g = f(1, 2, 3)
 h = g(2, 4, 6)
-verify(h() == 39)
+vereq(h(), 39)
 
 print "9. free variable in method"
 
@@ -141,9 +141,9 @@ def test():
     return Test()
 
 t = test()
-verify(t.test() == "var")
+vereq(t.test(), "var")
-verify(t.method_and_var() == "method")
+vereq(t.method_and_var(), "method")
-verify(t.actual_global() == "global")
+vereq(t.actual_global(), "global")
 
 method_and_var = "var"
 class Test:
@@ -158,9 +158,9 @@ class Test:
         return str(self)
 
 t = Test()
-verify(t.test() == "var")
+vereq(t.test(), "var")
-verify(t.method_and_var() == "method")
+vereq(t.method_and_var(), "method")
-verify(t.actual_global() == "global")
+vereq(t.actual_global(), "global")
 
 print "10. recursion"
 
@@ -175,7 +175,7 @@ def f(x):
     else:
         raise ValueError, "x must be >= 0"
 
-verify(f(6) == 720)
+vereq(f(6), 720)
 
 
 print "11. unoptimized namespaces"
@@ -252,24 +252,24 @@ print "12. lambdas"
 f1 = lambda x: lambda y: x + y
 inc = f1(1)
 plus10 = f1(10)
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(5) == 15)
+vereq(plus10(5), 15)
 
 f2 = lambda x: (lambda : lambda y: x + y)()
 inc = f2(1)
 plus10 = f2(10)
-verify(inc(1) == 2)
+vereq(inc(1), 2)
-verify(plus10(5) == 15)
+vereq(plus10(5), 15)
 
 f3 = lambda x: lambda y: global_x + y
 global_x = 1
 inc = f3(None)
-verify(inc(2) == 3)
+vereq(inc(2), 3)
 
 f8 = lambda x, y, z: lambda a, b, c: lambda : z * (b + y)
 g = f8(1, 2, 3)
 h = g(2, 4, 6)
-verify(h() == 18)
+vereq(h(), 18)
 
 print "13. UnboundLocal"
 
@@ -306,21 +306,21 @@ def makeReturner(*lst):
         return lst
     return returner
 
-verify(makeReturner(1,2,3)() == (1,2,3))
+vereq(makeReturner(1,2,3)(), (1,2,3))
 
 def makeReturner2(**kwargs):
     def returner():
         return kwargs
     return returner
 
-verify(makeReturner2(a=11)()['a'] == 11)
+vereq(makeReturner2(a=11)()['a'], 11)
 
 def makeAddPair((a, b)):
     def addPair((c, d)):
         return (a + c, b + d)
     return addPair
 
-verify(makeAddPair((1, 2))((100, 200)) == (101,202))
+vereq(makeAddPair((1, 2))((100, 200)), (101,202))
 
 print "15. scope of global statements"
 # Examples posted by Samuele Pedroni to python-dev on 3/1/2001
@@ -337,8 +337,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 7)
+vereq(f(), 7)
-verify(x == 7)
+vereq(x, 7)
 
 # II
 x = 7
@@ -352,8 +352,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
+vereq(f(), 2)
-verify(x == 7)
+vereq(x, 7)
 
 # III
 x = 7
@@ -368,8 +368,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
+vereq(f(), 2)
-verify(x == 2)
+vereq(x, 2)
 
 # IV
 x = 7
@@ -384,8 +384,25 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
+vereq(f(), 2)
-verify(x == 2)
+vereq(x, 2)
+
+# XXX what about global statements in class blocks?
+# do they affect methods?
+
+x = 12
+class Global:
+    global x
+    x = 13
+    def set(self, val):
+        x = val
+    def get(self):
+        return x
+
+g = Global()
+vereq(g.get(), 13)
+g.set(15)
+vereq(g.get(), 13)
 
 print "16. check leaks"
 
@@ -407,7 +424,7 @@ def f1():
 for i in range(100):
     f1()
 
-verify(Foo.count == 0)
+vereq(Foo.count, 0)
 
 print "17. class and global"
 
@@ -419,9 +436,9 @@ def test(x):
     return Foo()
 
 x = 0
-verify(test(6)(2) == 8)
+vereq(test(6)(2), 8)
 x = -1
-verify(test(3)(2) == 5)
+vereq(test(3)(2), 5)
 
 print "18. verify that locals() works"
 
@@ -437,7 +454,7 @@ def f(x):
 d = f(2)(4)
 verify(d.has_key('h'))
 del d['h']
-verify(d == {'x': 2, 'y': 7, 'w': 6})
+vereq(d, {'x': 2, 'y': 7, 'w': 6})
 
 print "19. var is bound and free in class"
 
@@ -449,7 +466,7 @@ def f(x):
     return C
 
 inst = f(3)()
-verify(inst.a == inst.m())
+vereq(inst.a, inst.m())
 
 print "20. interaction with trace function"
 

Makefile.pre.in

@@ -216,10 +216,22 @@ PGOBJS=		\
 
 PGENOBJS=	$(PGENMAIN) $(POBJS) $(PGOBJS)
 
+##########################################################################
+# AST
+AST_H=		$(srcdir)/Include/Python-ast.h
+AST_C=		$(srcdir)/Python/Python-ast.c
+AST_ASDL=	$(srcdir)/Parser/Python.asdl
+
+ASDLGEN_FILES=	$(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py
+# XXX Note that a build now requires Python exist before the build starts
+ASDLGEN=	$(srcdir)/Parser/asdl_c.py -h $(srcdir)/Include -c $(srcdir)/Python
 
 ##########################################################################
 # Python
 PYTHON_OBJS=	\
+		Python/Python-ast.o \
+		Python/asdl.o \
+		Python/ast.o \
 		Python/bltinmodule.o \
 		Python/exceptions.o \
 		Python/ceval.o \
@@ -265,6 +277,7 @@ OBJECT_OBJS=	\
 		Objects/cellobject.o \
 		Objects/classobject.o \
 		Objects/cobject.o \
+		Objects/codeobject.o \
 		Objects/complexobject.o \
 		Objects/descrobject.o \
 		Objects/enumobject.o \
@@ -457,8 +470,10 @@ Parser/metagrammar.o:	$(srcdir)/Parser/metagrammar.c
 
 Parser/tokenizer_pgen.o:	$(srcdir)/Parser/tokenizer.c
 
+$(AST_H) $(AST_C): $(AST_ASDL) $(ASDLGEN_FILES)
+	$(ASDLGEN) $(AST_ASDL)
 
-Python/compile.o Python/symtable.o: $(GRAMMAR_H)
+Python/compile.o Python/symtable.o: $(GRAMMAR_H) $(AST_H)
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -474,12 +489,15 @@ Objects/unicodectype.o:	$(srcdir)/Objects/unicodectype.c \
 
 PYTHON_HEADERS= \
 		Include/Python.h \
+		Include/Python-ast.h \
+		Include/asdl.h \
 		Include/abstract.h \
 		Include/boolobject.h \
 		Include/bufferobject.h \
 		Include/ceval.h \
 		Include/classobject.h \
 		Include/cobject.h \
+		Include/code.h \
 		Include/codecs.h \
 		Include/compile.h \
 		Include/complexobject.h \

Misc/ACKS

@@ -165,6 +165,7 @@ Eugene Dvurechenski
 Maxim Dzumanenko
 Hans Eckardt
 Grant Edwards
+John Ehresman
 Andrew Eland
 Lance Ellinghaus
 David Ely

Modules/_hotshot.c

@@ -3,6 +3,7 @@
  */
 
 #include "Python.h"
+#include "code.h"
 #include "compile.h"
 #include "eval.h"
 #include "frameobject.h"

Modules/symtablemodule.c

@@ -1,6 +1,8 @@
 #include "Python.h"
 
+#include "code.h"
 #include "compile.h"
+#include "Python-ast.h"
 #include "symtable.h"
 
 static PyObject *
@@ -64,9 +66,9 @@ init_symtable(void)
 	PyModule_AddIntConstant(m, "DEF_IMPORT", DEF_IMPORT);
 	PyModule_AddIntConstant(m, "DEF_BOUND", DEF_BOUND);
 
-	PyModule_AddIntConstant(m, "TYPE_FUNCTION", TYPE_FUNCTION);
+	PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock);
-	PyModule_AddIntConstant(m, "TYPE_CLASS", TYPE_CLASS);
+	PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock);
-	PyModule_AddIntConstant(m, "TYPE_MODULE", TYPE_MODULE);
+	PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock);
 
 	PyModule_AddIntConstant(m, "OPT_IMPORT_STAR", OPT_IMPORT_STAR);
 	PyModule_AddIntConstant(m, "OPT_EXEC", OPT_EXEC);

Objects/codeobject.c

@@ -0,0 +1,453 @@
+#include "Python.h"
+#include "code.h"
+#include "structmember.h"
+
+#define NAME_CHARS \
+	"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
+
+/* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */
+
+static int
+all_name_chars(unsigned char *s)
+{
+	static char ok_name_char[256];
+	static unsigned char *name_chars = (unsigned char *)NAME_CHARS;
+
+	if (ok_name_char[*name_chars] == 0) {
+		unsigned char *p;
+		for (p = name_chars; *p; p++)
+			ok_name_char[*p] = 1;
+	}
+	while (*s) {
+		if (ok_name_char[*s++] == 0)
+			return 0;
+	}
+	return 1;
+}
+
+static void
+intern_strings(PyObject *tuple)
+{
+	int i;
+
+	for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
+		PyObject *v = PyTuple_GET_ITEM(tuple, i);
+		if (v == NULL || !PyString_CheckExact(v)) {
+			Py_FatalError("non-string found in code slot");
+		}
+		PyString_InternInPlace(&PyTuple_GET_ITEM(tuple, i));
+	}
+}
+
+
+PyCodeObject *
+PyCode_New(int argcount, int nlocals, int stacksize, int flags,
+	   PyObject *code, PyObject *consts, PyObject *names,
+	   PyObject *varnames, PyObject *freevars, PyObject *cellvars,
+	   PyObject *filename, PyObject *name, int firstlineno,
+	   PyObject *lnotab)
+{
+	PyCodeObject *co;
+	int i;
+	/* Check argument types */
+	if (argcount < 0 || nlocals < 0 ||
+	    code == NULL ||
+	    consts == NULL || !PyTuple_Check(consts) ||
+	    names == NULL || !PyTuple_Check(names) ||
+	    varnames == NULL || !PyTuple_Check(varnames) ||
+	    freevars == NULL || !PyTuple_Check(freevars) ||
+	    cellvars == NULL || !PyTuple_Check(cellvars) ||
+	    name == NULL || !PyString_Check(name) ||
+	    filename == NULL || !PyString_Check(filename) ||
+	    lnotab == NULL || !PyString_Check(lnotab) ||
+	    !PyObject_CheckReadBuffer(code)) {
+		PyErr_BadInternalCall();
+		return NULL;
+	}
+	intern_strings(names);
+	intern_strings(varnames);
+	intern_strings(freevars);
+	intern_strings(cellvars);
+	/* Intern selected string constants */
+	for (i = PyTuple_Size(consts); --i >= 0; ) {
+		PyObject *v = PyTuple_GetItem(consts, i);
+		if (!PyString_Check(v))
+			continue;
+		if (!all_name_chars((unsigned char *)PyString_AS_STRING(v)))
+			continue;
+		PyString_InternInPlace(&PyTuple_GET_ITEM(consts, i));
+	}
+	co = PyObject_NEW(PyCodeObject, &PyCode_Type);
+	if (co != NULL) {
+		co->co_argcount = argcount;
+		co->co_nlocals = nlocals;
+		co->co_stacksize = stacksize;
+		co->co_flags = flags;
+		Py_INCREF(code);
+		co->co_code = code;
+		Py_INCREF(consts);
+		co->co_consts = consts;
+		Py_INCREF(names);
+		co->co_names = names;
+		Py_INCREF(varnames);
+		co->co_varnames = varnames;
+		Py_INCREF(freevars);
+		co->co_freevars = freevars;
+		Py_INCREF(cellvars);
+		co->co_cellvars = cellvars;
+		Py_INCREF(filename);
+		co->co_filename = filename;
+		Py_INCREF(name);
+		co->co_name = name;
+		co->co_firstlineno = firstlineno;
+		Py_INCREF(lnotab);
+		co->co_lnotab = lnotab;
+	}
+	return co;
+}
+
+
+#define OFF(x) offsetof(PyCodeObject, x)
+
+static PyMemberDef code_memberlist[] = {
+	{"co_argcount",	T_INT,		OFF(co_argcount),	READONLY},
+	{"co_nlocals",	T_INT,		OFF(co_nlocals),	READONLY},
+	{"co_stacksize",T_INT,		OFF(co_stacksize),	READONLY},
+	{"co_flags",	T_INT,		OFF(co_flags),		READONLY},
+	{"co_code",	T_OBJECT,	OFF(co_code),		READONLY},
+	{"co_consts",	T_OBJECT,	OFF(co_consts),		READONLY},
+	{"co_names",	T_OBJECT,	OFF(co_names),		READONLY},
+	{"co_varnames",	T_OBJECT,	OFF(co_varnames),	READONLY},
+	{"co_freevars",	T_OBJECT,	OFF(co_freevars),	READONLY},
+	{"co_cellvars",	T_OBJECT,	OFF(co_cellvars),	READONLY},
+	{"co_filename",	T_OBJECT,	OFF(co_filename),	READONLY},
+	{"co_name",	T_OBJECT,	OFF(co_name),		READONLY},
+	{"co_firstlineno", T_INT,	OFF(co_firstlineno),	READONLY},
+	{"co_lnotab",	T_OBJECT,	OFF(co_lnotab),		READONLY},
+	{NULL}	/* Sentinel */
+};
+
+/* Helper for code_new: return a shallow copy of a tuple that is
+   guaranteed to contain exact strings, by converting string subclasses
+   to exact strings and complaining if a non-string is found. */
+static PyObject*
+validate_and_copy_tuple(PyObject *tup)
+{
+	PyObject *newtuple;
+	PyObject *item;
+	int i, len;
+
+	len = PyTuple_GET_SIZE(tup);
+	newtuple = PyTuple_New(len);
+	if (newtuple == NULL)
+		return NULL;
+
+	for (i = 0; i < len; i++) {
+		item = PyTuple_GET_ITEM(tup, i);
+		if (PyString_CheckExact(item)) {
+			Py_INCREF(item);
+		}
+		else if (!PyString_Check(item)) {
+			PyErr_Format(
+				PyExc_TypeError,
+				"name tuples must contain only "
+				"strings, not '%.500s'",
+				item->ob_type->tp_name);
+			Py_DECREF(newtuple);
+			return NULL;
+		}
+		else {
+			item = PyString_FromStringAndSize(
+				PyString_AS_STRING(item),
+				PyString_GET_SIZE(item));
+			if (item == NULL) {
+				Py_DECREF(newtuple);
+				return NULL;
+			}
+		}
+		PyTuple_SET_ITEM(newtuple, i, item);
+	}
+
+	return newtuple;
+}
+
+PyDoc_STRVAR(code_doc,
+"code(argcount, nlocals, stacksize, flags, codestring, constants, names,\n\
+      varnames, filename, name, firstlineno, lnotab[, freevars[, cellvars]])\n\
+\n\
+Create a code object.  Not for the faint of heart.");
+
+static PyObject *
+code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
+{
+	int argcount;
+	int nlocals;
+	int stacksize;
+	int flags;
+	PyObject *co = NULL;
+	PyObject *code;
+	PyObject *consts;
+	PyObject *names, *ournames = NULL;
+	PyObject *varnames, *ourvarnames = NULL;
+	PyObject *freevars = NULL, *ourfreevars = NULL;
+	PyObject *cellvars = NULL, *ourcellvars = NULL;
+	PyObject *filename;
+	PyObject *name;
+	int firstlineno;
+	PyObject *lnotab;
+
+	if (!PyArg_ParseTuple(args, "iiiiSO!O!O!SSiS|O!O!:code",
+			      &argcount, &nlocals, &stacksize, &flags,
+			      &code,
+			      &PyTuple_Type, &consts,
+			      &PyTuple_Type, &names,
+			      &PyTuple_Type, &varnames,
+			      &filename, &name,
+			      &firstlineno, &lnotab,
+			      &PyTuple_Type, &freevars,
+			      &PyTuple_Type, &cellvars))
+		return NULL;
+
+	if (argcount < 0) {
+		PyErr_SetString(
+			PyExc_ValueError,
+			"code: argcount must not be negative");
+		goto cleanup;
+	}
+
+	if (nlocals < 0) {
+		PyErr_SetString(
+			PyExc_ValueError,
+			"code: nlocals must not be negative");
+		goto cleanup;
+	}
+
+	ournames = validate_and_copy_tuple(names);
+	if (ournames == NULL)
+		goto cleanup;
+	ourvarnames = validate_and_copy_tuple(varnames);
+	if (ourvarnames == NULL)
+		goto cleanup;
+	if (freevars)
+		ourfreevars = validate_and_copy_tuple(freevars);
+	else
+		ourfreevars = PyTuple_New(0);
+	if (ourfreevars == NULL)
+		goto cleanup;
+	if (cellvars)
+		ourcellvars = validate_and_copy_tuple(cellvars);
+	else
+		ourcellvars = PyTuple_New(0);
+	if (ourcellvars == NULL)
+		goto cleanup;
+
+	co = (PyObject *)PyCode_New(argcount, nlocals, stacksize, flags,
+				    code, consts, ournames, ourvarnames,
+				    ourfreevars, ourcellvars, filename,
+				    name, firstlineno, lnotab);
+  cleanup:
+	Py_XDECREF(ournames);
+	Py_XDECREF(ourvarnames);
+	Py_XDECREF(ourfreevars);
+	Py_XDECREF(ourcellvars);
+	return co;
+}
+
+static void
+code_dealloc(PyCodeObject *co)
+{
+	Py_XDECREF(co->co_code);
+	Py_XDECREF(co->co_consts);
+	Py_XDECREF(co->co_names);
+	Py_XDECREF(co->co_varnames);
+	Py_XDECREF(co->co_freevars);
+	Py_XDECREF(co->co_cellvars);
+	Py_XDECREF(co->co_filename);
+	Py_XDECREF(co->co_name);
+	Py_XDECREF(co->co_lnotab);
+	PyObject_DEL(co);
+}
+
+static PyObject *
+code_repr(PyCodeObject *co)
+{
+	char buf[500];
+	int lineno = -1;
+	char *filename = "???";
+	char *name = "???";
+
+	if (co->co_firstlineno != 0)
+		lineno = co->co_firstlineno;
+	if (co->co_filename && PyString_Check(co->co_filename))
+		filename = PyString_AS_STRING(co->co_filename);
+	if (co->co_name && PyString_Check(co->co_name))
+		name = PyString_AS_STRING(co->co_name);
+	PyOS_snprintf(buf, sizeof(buf),
+		      "<code object %.100s at %p, file \"%.300s\", line %d>",
+		      name, co, filename, lineno);
+	return PyString_FromString(buf);
+}
+
+static int
+code_compare(PyCodeObject *co, PyCodeObject *cp)
+{
+	int cmp;
+	cmp = PyObject_Compare(co->co_name, cp->co_name);
+	if (cmp) return cmp;
+	cmp = co->co_argcount - cp->co_argcount;
+	if (cmp) goto normalize;
+	cmp = co->co_nlocals - cp->co_nlocals;
+	if (cmp) goto normalize;
+	cmp = co->co_flags - cp->co_flags;
+	if (cmp) goto normalize;
+	cmp = co->co_firstlineno - cp->co_firstlineno;
+	if (cmp) goto normalize;
+	cmp = PyObject_Compare(co->co_code, cp->co_code);
+	if (cmp) return cmp;
+	cmp = PyObject_Compare(co->co_consts, cp->co_consts);
+	if (cmp) return cmp;
+	cmp = PyObject_Compare(co->co_names, cp->co_names);
+	if (cmp) return cmp;
+	cmp = PyObject_Compare(co->co_varnames, cp->co_varnames);
+	if (cmp) return cmp;
+	cmp = PyObject_Compare(co->co_freevars, cp->co_freevars);
+	if (cmp) return cmp;
+	cmp = PyObject_Compare(co->co_cellvars, cp->co_cellvars);
+	return cmp;
+
+ normalize:
+	if (cmp > 0)
+		return 1;
+	else if (cmp < 0)
+		return -1;
+	else
+		return 0;
+}
+
+static long
+code_hash(PyCodeObject *co)
+{
+	long h, h0, h1, h2, h3, h4, h5, h6;
+	h0 = PyObject_Hash(co->co_name);
+	if (h0 == -1) return -1;
+	h1 = PyObject_Hash(co->co_code);
+	if (h1 == -1) return -1;
+	h2 = PyObject_Hash(co->co_consts);
+	if (h2 == -1) return -1;
+	h3 = PyObject_Hash(co->co_names);
+	if (h3 == -1) return -1;
+	h4 = PyObject_Hash(co->co_varnames);
+	if (h4 == -1) return -1;
+	h5 = PyObject_Hash(co->co_freevars);
+	if (h5 == -1) return -1;
+	h6 = PyObject_Hash(co->co_cellvars);
+	if (h6 == -1) return -1;
+	h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^
+		co->co_argcount ^ co->co_nlocals ^ co->co_flags;
+	if (h == -1) h = -2;
+	return h;
+}
+
+/* XXX code objects need to participate in GC? */
+
+PyTypeObject PyCode_Type = {
+	PyObject_HEAD_INIT(&PyType_Type)
+	0,
+	"code",
+	sizeof(PyCodeObject),
+	0,
+	(destructor)code_dealloc, 	/* tp_dealloc */
+	0,				/* tp_print */
+	0, 				/* tp_getattr */
+	0,				/* tp_setattr */
+	(cmpfunc)code_compare, 		/* tp_compare */
+	(reprfunc)code_repr,		/* tp_repr */
+	0,				/* tp_as_number */
+	0,				/* tp_as_sequence */
+	0,				/* tp_as_mapping */
+	(hashfunc)code_hash, 		/* tp_hash */
+	0,				/* tp_call */
+	0,				/* tp_str */
+	PyObject_GenericGetAttr,	/* tp_getattro */
+	0,				/* tp_setattro */
+	0,				/* tp_as_buffer */
+	Py_TPFLAGS_DEFAULT,		/* tp_flags */
+	code_doc,			/* tp_doc */
+	0,				/* tp_traverse */
+	0,				/* tp_clear */
+	0,				/* tp_richcompare */
+	0,				/* tp_weaklistoffset */
+	0,				/* tp_iter */
+	0,				/* tp_iternext */
+	0,				/* tp_methods */
+	code_memberlist,		/* tp_members */
+	0,				/* tp_getset */
+	0,				/* tp_base */
+	0,				/* tp_dict */
+	0,				/* tp_descr_get */
+	0,				/* tp_descr_set */
+	0,				/* tp_dictoffset */
+	0,				/* tp_init */
+	0,				/* tp_alloc */
+	code_new,			/* tp_new */
+};
+
+/* All about c_lnotab.
+
+c_lnotab is an array of unsigned bytes disguised as a Python string.  In -O
+mode, SET_LINENO opcodes aren't generated, and bytecode offsets are mapped
+to source code line #s (when needed for tracebacks) via c_lnotab instead.
+The array is conceptually a list of
+    (bytecode offset increment, line number increment)
+pairs.  The details are important and delicate, best illustrated by example:
+
+    byte code offset    source code line number
+        0		    1
+        6		    2
+       50		    7
+      350                 307
+      361                 308
+
+The first trick is that these numbers aren't stored, only the increments
+from one row to the next (this doesn't really work, but it's a start):
+
+    0, 1,  6, 1,  44, 5,  300, 300,  11, 1
+
+The second trick is that an unsigned byte can't hold negative values, or
+values larger than 255, so (a) there's a deep assumption that byte code
+offsets and their corresponding line #s both increase monotonically, and (b)
+if at least one column jumps by more than 255 from one row to the next, more
+than one pair is written to the table. In case #b, there's no way to know
+from looking at the table later how many were written.  That's the delicate
+part.  A user of c_lnotab desiring to find the source line number
+corresponding to a bytecode address A should do something like this
+
+    lineno = addr = 0
+    for addr_incr, line_incr in c_lnotab:
+        addr += addr_incr
+        if addr > A:
+            return lineno
+        lineno += line_incr
+
+In order for this to work, when the addr field increments by more than 255,
+the line # increment in each pair generated must be 0 until the remaining addr
+increment is < 256.  So, in the example above, com_set_lineno should not (as
+was actually done until 2.2) expand 300, 300 to 255, 255,  45, 45, but to
+255, 0,  45, 255,  0, 45.
+*/
+
+int
+PyCode_Addr2Line(PyCodeObject *co, int addrq)
+{
+	int size = PyString_Size(co->co_lnotab) / 2;
+	unsigned char *p = (unsigned char*)PyString_AsString(co->co_lnotab);
+	int line = co->co_firstlineno;
+	int addr = 0;
+	while (--size >= 0) {
+		addr += *p++;
+		if (addr > addrq)
+			break;
+		line += *p++;
+	}
+	return line;
+}

Objects/frameobject.c

@@ -3,6 +3,7 @@
 
 #include "Python.h"
 
+#include "code.h"
 #include "compile.h"
 #include "frameobject.h"
 #include "opcode.h"

Objects/funcobject.c

@@ -2,7 +2,7 @@
 /* Function object implementation */
 
 #include "Python.h"
-#include "compile.h"
+#include "code.h"
 #include "eval.h"
 #include "structmember.h"
 
@@ -144,7 +144,9 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure)
 		Py_XINCREF(closure);
 	}
 	else {
-		PyErr_SetString(PyExc_SystemError, "non-tuple closure");
+		PyErr_Format(PyExc_SystemError, 
+			     "expected tuple for closure, got '%.100s'",
+			     closure->ob_type->tp_name);
 		return -1;
 	}
 	Py_XDECREF(((PyFunctionObject *) op) -> func_closure);

Objects/typeobject.c

@@ -1737,20 +1737,14 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
 			goto bad_slots;
 		for (i = j = 0; i < nslots; i++) {
 			char *s;
-			char buffer[256];
 			tmp = PyTuple_GET_ITEM(slots, i);
 			s = PyString_AS_STRING(tmp);
 			if ((add_dict && strcmp(s, "__dict__") == 0) ||
 			    (add_weak && strcmp(s, "__weakref__") == 0))
 				continue;
-			if (_Py_Mangle(PyString_AS_STRING(name),
+			tmp =_Py_Mangle(name, tmp);
-				       PyString_AS_STRING(tmp),
+                        if (!tmp)
-				       buffer, sizeof(buffer)))
+                            goto bad_slots;
-			{
-				tmp = PyString_FromString(buffer);
-			} else {
-				Py_INCREF(tmp);
-			}
 			PyTuple_SET_ITEM(newslots, j, tmp);
 			j++;
 		}

PCbuild/pythoncore.vcproj

@@ -3,6 +3,7 @@
 	ProjectType="Visual C++"
 	Version="7.10"
 	Name="pythoncore"
+	ProjectGUID="{7AFA1F0B-A8A1-455A-A832-BF263404BBEF}"
 	SccProjectName="pythoncore"
 	SccLocalPath="..">
 	<Platforms>
@@ -477,6 +478,12 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Python\asdl.c">
+		</File>
+		<File
+			RelativePath="..\Python\ast.c">
+		</File>
 		<File
 			RelativePath="..\Modules\audioop.c">
 			<FileConfiguration
@@ -802,34 +809,10 @@
 			</FileConfiguration>
 		</File>
 		<File
-			RelativePath="..\Modules\collectionsmodule.c">
+			RelativePath="..\Objects\codeobject.c">
 		</File>
 		<File
-			RelativePath="..\Python\compile.c">
+			RelativePath="..\Modules\collectionsmodule.c">
-			<FileConfiguration
-				Name="Release|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="2"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
-			</FileConfiguration>
-			<FileConfiguration
-				Name="Debug|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="0"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
-			</FileConfiguration>
-			<FileConfiguration
-				Name="ReleaseItanium|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="2"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
-			</FileConfiguration>
 		</File>
 		<File
 			RelativePath="..\Objects\complexobject.c">
@@ -1185,6 +1168,9 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Parser\firstsets.c">
+		</File>
 		<File
 			RelativePath="..\Objects\floatobject.c">
 			<FileConfiguration
@@ -1320,6 +1306,33 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Modules\functionalmodule.c">
+			<FileConfiguration
+				Name="Release|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="Debug|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="0"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="ReleaseItanium|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+		</File>
 		<File
 			RelativePath="..\Python\future.c">
 			<FileConfiguration
@@ -1647,6 +1660,9 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Parser\grammar.c">
+		</File>
 		<File
 			RelativePath="..\Parser\grammar1.c">
 			<FileConfiguration
@@ -2301,6 +2317,9 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Python\compile.c">
+		</File>
 		<File
 			RelativePath="..\Parser\node.c">
 			<FileConfiguration
@@ -2436,9 +2455,6 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
-		<File
-			RelativePath="..\Modules\parsermodule.c">
-		</File>
 		<File
 			RelativePath="..\Parser\parsetok.c">
 			<FileConfiguration
@@ -2466,6 +2482,9 @@
 					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
 			</FileConfiguration>
 		</File>
+		<File
+			RelativePath="..\Parser\pgen.c">
+		</File>
 		<File
 			RelativePath="..\Modules\posixmodule.c">
 			<FileConfiguration
@@ -2550,6 +2569,9 @@
 		<File
 			RelativePath="..\Python\pystrtod.c">
 		</File>
+		<File
+			RelativePath="..\Python\Python-ast.c">
+		</File>
 		<File
 			RelativePath="..\PC\python_nt.rc">
 			<FileConfiguration
@@ -2764,6 +2786,7 @@
 			</FileConfiguration>
 		</File>
 		<File
+<<<<<<< pythoncore.vcproj
                         RelativePath="..\Modules\sha256module.c">
 			<FileConfiguration
 				Name="Release|Win32">
@@ -2819,6 +2842,63 @@
 		</File>
 		<File
 			RelativePath="..\Modules\signalmodule.c">
+=======
+                        RelativePath="..\Modules\sha256module.c">
+>>>>>>> 1.26.2.3
+			<FileConfiguration
+				Name="Release|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="Debug|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="0"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="ReleaseItanium|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+		</File>
+		<File
+                        RelativePath="..\Modules\sha512module.c">
+			<FileConfiguration
+				Name="Release|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="Debug|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="0"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
+			</FileConfiguration>
+			<FileConfiguration
+				Name="ReleaseItanium|Win32">
+				<Tool
+					Name="VCCLCompilerTool"
+					Optimization="2"
+					AdditionalIncludeDirectories=""
+					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
+			</FileConfiguration>
+		</File>
+		<File
+			RelativePath="..\Modules\signalmodule.c">
 			<FileConfiguration
 				Name="Release|Win32">
 				<Tool
@@ -3145,31 +3225,7 @@
 			</FileConfiguration>
 		</File>
 		<File
-			RelativePath="..\Parser\tokenizer.c">
+			RelativePath="..\Parser\tokenizer_pgen.c">
-			<FileConfiguration
-				Name="Release|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="2"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
-			</FileConfiguration>
-			<FileConfiguration
-				Name="Debug|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="0"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
-			</FileConfiguration>
-			<FileConfiguration
-				Name="ReleaseItanium|Win32">
-				<Tool
-					Name="VCCLCompilerTool"
-					Optimization="2"
-					AdditionalIncludeDirectories=""
-					PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
-			</FileConfiguration>
 		</File>
 		<File
 			RelativePath="..\Python\traceback.c">

Parser/.cvsignore

@@ -1,3 +1,6 @@
 Makefile
 pgen
 add2lib
+asdl.pyc
+asdl_c.pyc
+spark.pyc

Parser/Python.asdl

@@ -0,0 +1,107 @@
+-- ASDL's three builtin types are identifier, int, string
+
+module Python
+{
+	mod = Module(stmt* body)
+	    | Interactive(stmt* body)
+	    | Expression(expr body)
+
+	    -- not really an actual node but useful in Jython's typesystem.
+	    | Suite(stmt* body)
+
+	stmt = FunctionDef(identifier name, arguments args, 
+                           stmt* body, expr* decorators)
+	      | ClassDef(identifier name, expr* bases, stmt* body)
+	      | Return(expr? value)
+
+	      | Delete(expr* targets)
+	      | Assign(expr* targets, expr value)
+	      | AugAssign(expr target, operator op, expr value)
+
+	      -- not sure if bool is allowed, can always use int
+ 	      | Print(expr? dest, expr* values, bool nl)
+
+	      -- use 'orelse' because else is a keyword in target languages
+	      | For(expr target, expr iter, stmt* body, stmt* orelse)
+	      | While(expr test, stmt* body, stmt* orelse)
+	      | If(expr test, stmt* body, stmt* orelse)
+
+	      -- 'type' is a bad name
+	      | Raise(expr? type, expr? inst, expr? tback)
+	      | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
+	      | TryFinally(stmt* body, stmt* finalbody)
+	      | Assert(expr test, expr? msg)
+
+	      | Import(alias* names)
+	      | ImportFrom(identifier module, alias* names)
+
+	      -- Doesn't capture requirement that locals must be
+	      -- defined if globals is
+	      -- still supports use as a function!
+	      | Exec(expr body, expr? globals, expr? locals)
+
+	      | Global(identifier* names)
+	      | Expr(expr value)
+	      | Pass | Break | Continue
+
+	      -- XXX Jython will be different
+	      attributes (int lineno)
+
+	      -- BoolOp() can use left & right?
+	expr = BoolOp(boolop op, expr* values)
+	     | BinOp(expr left, operator op, expr right)
+	     | UnaryOp(unaryop op, expr operand)
+	     | Lambda(arguments args, expr body)
+	     | Dict(expr* keys, expr* values)
+	     | ListComp(expr elt, comprehension* generators)
+	     | GeneratorExp(expr elt, comprehension* generators)
+	     | Yield(expr? value)
+	     -- need sequences for compare to distinguish between
+	     -- x < 4 < 3 and (x < 4) < 3
+	     | Compare(expr left, cmpop* ops, expr* comparators)
+	     | Call(expr func, expr* args, keyword* keywords,
+			 expr? starargs, expr? kwargs)
+	     | Repr(expr value)
+	     | Num(object n) -- a number as a PyObject.
+	     | Str(string s) -- need to specify raw, unicode, etc?
+	     -- other literals? bools?
+
+	     -- the following expression can appear in assignment context
+	     | Attribute(expr value, identifier attr, expr_context ctx)
+	     | Subscript(expr value, slice slice, expr_context ctx)
+	     | Name(identifier id, expr_context ctx)
+	     | List(expr* elts, expr_context ctx) 
+	     | Tuple(expr *elts, expr_context ctx)
+
+	      attributes (int lineno)
+
+	expr_context = Load | Store | Del | AugLoad | AugStore | Param
+
+	slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) 
+	      | ExtSlice(slice* dims) 
+	      | Index(expr value) 
+
+	boolop = And | Or 
+
+	operator = Add | Sub | Mult | Div | Mod | Pow | LShift 
+                 | RShift | BitOr | BitXor | BitAnd | FloorDiv
+
+	unaryop = Invert | Not | UAdd | USub
+
+	cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
+
+	comprehension = (expr target, expr iter, expr* ifs)
+
+	-- not sure what to call the first argument for raise and except
+
+	excepthandler = (expr? type, expr? name, stmt* body)
+
+	arguments = (expr* args, identifier? vararg, 
+		     identifier? kwarg, expr* defaults)
+
+        -- keyword arguments supplied to call
+        keyword = (identifier arg, expr value)
+
+        -- import name with optional 'as' alias.
+        alias = (identifier name, identifier? asname)
+}

Parser/asdl.py

@@ -0,0 +1,393 @@
+"""An implementation of the Zephyr Abstract Syntax Definition Language.
+
+See http://asdl.sourceforge.net/ and
+http://www.cs.princeton.edu/~danwang/Papers/dsl97/dsl97-abstract.html.
+
+Only supports top level module decl, not view.  I'm guessing that view
+is intended to support the browser and I'm not interested in the
+browser.
+"""
+
+#__metaclass__ = type
+
+import os
+import traceback
+
+import spark
+
+class Token:
+    # spark seems to dispatch in the parser based on a token's
+    # type attribute
+    def __init__(self, type, lineno):
+        self.type = type
+        self.lineno = lineno
+
+    def __str__(self):
+        return self.type
+
+    def __repr__(self):
+        return str(self)
+
+class Id(Token):
+    def __init__(self, value, lineno):
+        self.type = 'Id'
+        self.value = value
+        self.lineno = lineno
+
+    def __str__(self):
+        return self.value
+
+class ASDLSyntaxError:
+
+    def __init__(self, lineno, token=None, msg=None):
+        self.lineno = lineno
+        self.token = token
+        self.msg = msg
+
+    def __str__(self):
+        if self.msg is None:
+            return "Error at '%s', line %d" % (self.token, self.lineno)
+        else:
+            return "%s, line %d" % (self.msg, self.lineno)
+
+class ASDLScanner(spark.GenericScanner, object):
+
+    def tokenize(self, input):
+        self.rv = []
+        self.lineno = 1
+        super(ASDLScanner, self).tokenize(input)
+        return self.rv
+
+    def t_id(self, s):
+        r"[\w\.]+"
+        # XXX doesn't distinguish upper vs. lower, which is
+        # significant for ASDL.
+        self.rv.append(Id(s, self.lineno))
+
+    def t_xxx(self, s): # not sure what this production means
+        r"<="
+        self.rv.append(Token(s, self.lineno))
+
+    def t_punctuation(self, s):
+        r"[\{\}\*\=\|\(\)\,\?\:]"
+        self.rv.append(Token(s, self.lineno))
+
+    def t_comment(self, s):
+        r"\-\-[^\n]*"
+        pass
+
+    def t_newline(self, s):
+        r"\n"
+        self.lineno += 1
+
+    def t_whitespace(self, s):
+        r"[ \t]+"
+        pass
+
+    def t_default(self, s):
+        r" . +"
+        raise ValueError, "unmatched input: %s" % `s`
+
+class ASDLParser(spark.GenericParser, object):
+    def __init__(self):
+        super(ASDLParser, self).__init__("module")
+
+    def typestring(self, tok):
+        return tok.type
+
+    def error(self, tok):
+        raise ASDLSyntaxError(tok.lineno, tok)
+
+    def p_module_0(self, (module, name, _0, _1)):
+        " module ::= Id Id { } "
+        if module.value != "module":
+            raise ASDLSyntaxError(module.lineno,
+                                  msg="expected 'module', found %s" % module)
+        return Module(name, None)
+
+    def p_module(self, (module, name, _0, definitions, _1)):
+        " module ::= Id Id { definitions } "
+        if module.value != "module":
+            raise ASDLSyntaxError(module.lineno,
+                                  msg="expected 'module', found %s" % module)
+        return Module(name, definitions)
+
+    def p_definition_0(self, (definition,)):
+        " definitions ::= definition "
+        return definition
+
+    def p_definition_1(self, (definitions, definition)):
+        " definitions ::= definition definitions "
+        return definitions + definition
+
+    def p_definition(self, (id, _, type)):
+        " definition ::= Id = type "
+        return [Type(id, type)]
+
+    def p_type_0(self, (product,)):
+        " type ::= product "
+        return product
+
+    def p_type_1(self, (sum,)):
+        " type ::= sum "
+        return Sum(sum)
+
+    def p_type_2(self, (sum, id, _0, attributes, _1)):
+        " type ::= sum Id ( fields ) "
+        if id.value != "attributes":
+            raise ASDLSyntaxError(id.lineno,
+                                  msg="expected attributes, found %s" % id)
+        return Sum(sum, attributes)
+
+    def p_product(self, (_0, fields, _1)):
+        " product ::= ( fields ) "
+        # XXX can't I just construct things in the right order?
+        fields.reverse() 
+        return Product(fields)
+
+    def p_sum_0(self, (constructor,)):
+        " sum ::= constructor """
+        return [constructor]
+
+    def p_sum_1(self, (constructor, _, sum)):
+        " sum ::= constructor | sum "
+        return [constructor] + sum
+
+    def p_sum_2(self, (constructor, _, sum)):
+        " sum ::= constructor | sum "
+        return [constructor] + sum
+
+    def p_constructor_0(self, (id,)):
+        " constructor ::= Id "
+        return Constructor(id)
+
+    def p_constructor_1(self, (id, _0, fields, _1)):
+        " constructor ::= Id ( fields ) "
+        # XXX can't I just construct things in the right order?
+        fields.reverse() 
+        return Constructor(id, fields)
+
+    def p_fields_0(self, (field,)):
+        " fields ::= field "
+        return [field]
+
+    def p_fields_1(self, (field, _, fields)):
+        " fields ::= field , fields "
+        return fields + [field]
+
+    def p_field_0(self, (type,)):
+        " field ::= Id "
+        return Field(type)
+
+    def p_field_1(self, (type, name)):
+        " field ::= Id Id "
+        return Field(type, name)
+
+    def p_field_2(self, (type, _, name)):
+        " field ::= Id * Id "
+        return Field(type, name, seq=1)
+
+    def p_field_3(self, (type, _, name)):
+        " field ::= Id ? Id "
+        return Field(type, name, opt=1)
+
+    def p_field_4(self, (type, _)):
+        " field ::= Id * "
+        return Field(type, seq=1)
+
+    def p_field_5(self, (type, _)):
+        " field ::= Id ? "
+        return Field(type, opt=1)
+
+builtin_types = ("identifier", "string", "int", "bool", "object")
+
+# below is a collection of classes to capture the AST of an AST :-)
+# not sure if any of the methods are useful yet, but I'm adding them
+# piecemeal as they seem helpful
+
+class AST:
+    pass # a marker class
+
+class Module(AST):
+    def __init__(self, name, dfns):
+        self.name = name
+        self.dfns = dfns
+        self.types = {} # maps type name to value (from dfns)
+        for type in dfns:
+            self.types[type.name.value] = type.value
+
+    def __repr__(self):
+        return "Module(%s, %s)" % (self.name, self.dfns)
+
+class Type(AST):
+    def __init__(self, name, value):
+        self.name = name
+        self.value = value
+
+    def __repr__(self):
+        return "Type(%s, %s)" % (self.name, self.value)
+
+class Constructor(AST):
+    def __init__(self, name, fields=None):
+        self.name = name
+        self.fields = fields or []
+
+    def __repr__(self):
+        return "Constructor(%s, %s)" % (self.name, self.fields)
+
+class Field(AST):
+    def __init__(self, type, name=None, seq=0, opt=0):
+        self.type = type
+        self.name = name
+        self.seq = seq
+        self.opt = opt
+
+    def __repr__(self):
+        if self.seq:
+            extra = ", seq=1"
+        elif self.opt:
+            extra = ", opt=1"
+        else:
+            extra = ""
+        if self.name is None:
+            return "Field(%s%s)" % (self.type, extra)
+        else:
+            return "Field(%s, %s%s)" % (self.type, self.name, extra)
+
+class Sum(AST):
+    def __init__(self, types, attributes=None):
+        self.types = types
+        self.attributes = attributes or []
+
+    def __repr__(self):
+        if self.attributes is None:
+            return "Sum(%s)" % self.types
+        else:
+            return "Sum(%s, %s)" % (self.types, self.attributes)
+
+class Product(AST):
+    def __init__(self, fields):
+        self.fields = fields
+
+    def __repr__(self):
+        return "Product(%s)" % self.fields
+
+class VisitorBase(object):
+
+    def __init__(self, skip=0):
+        self.cache = {}
+        self.skip = skip
+
+    def visit(self, object, *args):
+        meth = self._dispatch(object)
+        if meth is None:
+            return
+        try:
+            meth(object, *args)
+        except Exception, err:
+            print "Error visiting", repr(object)
+            print err
+            traceback.print_exc()
+            # XXX hack
+            if hasattr(self, 'file'):
+                self.file.flush()
+            os._exit(1)
+
+    def _dispatch(self, object):
+        assert isinstance(object, AST), repr(object)
+        klass = object.__class__
+        meth = self.cache.get(klass)
+        if meth is None:
+            methname = "visit" + klass.__name__
+            if self.skip:
+                meth = getattr(self, methname, None)
+            else:
+                meth = getattr(self, methname)
+            self.cache[klass] = meth
+        return meth
+
+class Check(VisitorBase):
+
+    def __init__(self):
+        super(Check, self).__init__(skip=1)
+        self.cons = {}
+        self.errors = 0
+        self.types = {}
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type):
+        self.visit(type.value, str(type.name))
+
+    def visitSum(self, sum, name):
+        for t in sum.types:
+            self.visit(t, name)
+
+    def visitConstructor(self, cons, name):
+        key = str(cons.name)
+        conflict = self.cons.get(key)
+        if conflict is None:
+            self.cons[key] = name
+        else:
+            print "Redefinition of constructor %s" % key
+            print "Defined in %s and %s" % (conflict, name)
+            self.errors += 1
+        for f in cons.fields:
+            self.visit(f, key)
+
+    def visitField(self, field, name):
+        key = str(field.type)
+        l = self.types.setdefault(key, [])
+        l.append(name)
+
+    def visitProduct(self, prod, name):
+        for f in prod.fields:
+            self.visit(f, name)
+
+def check(mod):
+    v = Check()
+    v.visit(mod)
+
+    for t in v.types:
+        if not mod.types.has_key(t) and not t in builtin_types:
+            v.errors += 1
+            uses = ", ".join(v.types[t])
+            print "Undefined type %s, used in %s" % (t, uses)
+    
+    return not v.errors
+
+def parse(file):
+    scanner = ASDLScanner()
+    parser = ASDLParser()
+
+    buf = open(file).read()
+    tokens = scanner.tokenize(buf)
+    try:
+        return parser.parse(tokens)
+    except ASDLSyntaxError, err:
+        print err
+        lines = buf.split("\n")
+        print lines[err.lineno - 1] # lines starts at 0, files at 1
+
+if __name__ == "__main__":
+    import glob
+    import sys
+
+    if len(sys.argv) > 1:
+        files = sys.argv[1:]
+    else:
+        testdir = "tests"
+        files = glob.glob(testdir + "/*.asdl")
+    
+    for file in files:
+        print file
+        mod = parse(file)
+        print "module", mod.name
+        print len(mod.dfns), "definitions"
+        if not check(mod):
+            print "Check failed"
+        else:
+            for dfn in mod.dfns:
+                print dfn.type

Parser/asdl_c.py

@@ -0,0 +1,621 @@
+#! /usr/bin/env python
+"""Generate C code from an ASDL description."""
+
+# TO DO
+# handle fields that have a type but no name
+
+import os, sys, traceback
+
+import asdl
+
+TABSIZE = 8
+MAX_COL = 80
+
+def get_c_type(name):
+    """Return a string for the C name of the type.
+
+    This function special cases the default types provided by asdl:
+    identifier, string, int, bool.
+    """
+    # XXX ack!  need to figure out where Id is useful and where string
+    if isinstance(name, asdl.Id):
+        name = name.value
+    if name in asdl.builtin_types:
+        return name
+    else:
+        return "%s_ty" % name
+
+def reflow_lines(s, depth):
+    """Reflow the line s indented depth tabs.
+
+    Return a sequence of lines where no line extends beyond MAX_COL
+    when properly indented.  The first line is properly indented based
+    exclusively on depth * TABSIZE.  All following lines -- these are
+    the reflowed lines generated by this function -- start at the same
+    column as the first character beyond the opening { in the first
+    line.
+    """
+    size = MAX_COL - depth * TABSIZE
+    if len(s) < size:
+        return [s]
+
+    lines = []
+    cur = s
+    padding = ""
+    while len(cur) > size:
+        i = cur.rfind(' ', 0, size)
+        # XXX this should be fixed for real
+        if i == -1 and 'GeneratorExp' in cur:
+            i = size + 3
+        assert i != -1, "Impossible line %d to reflow: %s" % (size, `s`)
+        lines.append(padding + cur[:i])
+        if len(lines) == 1:
+            # find new size based on brace
+            j = cur.find('{', 0, i)
+            if j >= 0:
+                j += 2 # account for the brace and the space after it
+                size -= j
+                padding = " " * j
+            else:
+                j = cur.find('(', 0, i)
+                if j >= 0:
+                    j += 1 # account for the paren (no space after it)
+                    size -= j
+                    padding = " " * j
+        cur = cur[i+1:]
+    else:
+        lines.append(padding + cur)
+    return lines
+
+def is_simple(sum):
+    """Return True if a sum is a simple.
+
+    A sum is simple if its types have no fields, e.g.
+    unaryop = Invert | Not | UAdd | USub
+    """
+
+    for t in sum.types:
+        if t.fields:
+            return False
+    return True
+
+class EmitVisitor(asdl.VisitorBase):
+    """Visit that emits lines"""
+
+    def __init__(self, file):
+        self.file = file
+        super(EmitVisitor, self).__init__()
+
+    def emit(self, s, depth, reflow=1):
+        # XXX reflow long lines?
+        if reflow:
+            lines = reflow_lines(s, depth)
+        else:
+            lines = [s]
+        for line in lines:
+            line = (" " * TABSIZE * depth) + line + "\n"
+            self.file.write(line)
+
+class TypeDefVisitor(EmitVisitor):
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        if is_simple(sum):
+            self.simple_sum(sum, name, depth)
+        else:
+            self.sum_with_constructors(sum, name, depth)
+
+    def simple_sum(self, sum, name, depth):
+        enum = []
+        for i in range(len(sum.types)):
+            type = sum.types[i]
+            enum.append("%s=%d" % (type.name, i + 1))
+        enums = ", ".join(enum)
+        ctype = get_c_type(name)
+        s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
+        self.emit(s, depth)
+        self.emit("", depth)
+
+    def sum_with_constructors(self, sum, name, depth):
+        ctype = get_c_type(name)
+        s = "typedef struct _%(name)s *%(ctype)s;" % locals()
+        self.emit(s, depth)
+        self.emit("", depth)
+
+    def visitProduct(self, product, name, depth):
+        ctype = get_c_type(name)
+        s = "typedef struct _%(name)s *%(ctype)s;" % locals()
+        self.emit(s, depth)
+        self.emit("", depth)
+
+class StructVisitor(EmitVisitor):
+    """Visitor to generate typdefs for AST."""
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        if not is_simple(sum):
+            self.sum_with_constructors(sum, name, depth)
+
+    def sum_with_constructors(self, sum, name, depth):
+        def emit(s, depth=depth):
+            self.emit(s % sys._getframe(1).f_locals, depth)
+        enum = []
+        for i in range(len(sum.types)):
+            type = sum.types[i]
+            enum.append("%s_kind=%d" % (type.name, i + 1))
+
+        emit("struct _%(name)s {")
+        emit("enum { " + ", ".join(enum) + " } kind;", depth + 1)
+        emit("union {", depth + 1)
+        for t in sum.types:
+            self.visit(t, depth + 2)
+        emit("} v;", depth + 1)
+        for field in sum.attributes:
+            # rudimentary attribute handling
+            type = str(field.type)
+            assert type in asdl.builtin_types, type
+            emit("%s %s;" % (type, field.name), depth + 1);
+        emit("};")
+        emit("")
+
+    def visitConstructor(self, cons, depth):
+        if cons.fields:
+            self.emit("struct {", depth)
+            for f in cons.fields:
+                self.visit(f, depth + 1)
+            self.emit("} %s;" % cons.name, depth)
+            self.emit("", depth)
+        else:
+            # XXX not sure what I want here, nothing is probably fine
+            pass
+
+    def visitField(self, field, depth):
+        # XXX need to lookup field.type, because it might be something
+        # like a builtin...
+        ctype = get_c_type(field.type)
+        name = field.name
+        if field.seq:
+            self.emit("asdl_seq *%(name)s;" % locals(), depth)
+        else:
+            self.emit("%(ctype)s %(name)s;" % locals(), depth)
+
+    def visitProduct(self, product, name, depth):
+        self.emit("struct _%(name)s {" % locals(), depth)
+        for f in product.fields:
+            self.visit(f, depth + 1)
+        self.emit("};", depth)
+        self.emit("", depth)
+
+class PrototypeVisitor(EmitVisitor):
+    """Generate function prototypes for the .h file"""
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type):
+        self.visit(type.value, type.name)
+
+    def visitSum(self, sum, name):
+        if is_simple(sum):
+            pass # XXX
+        else:
+            for t in sum.types:
+                self.visit(t, name, sum.attributes)
+
+    def get_args(self, fields):
+        """Return list of C argument into, one for each field.
+
+        Argument info is 3-tuple of a C type, variable name, and flag
+        that is true if type can be NULL.
+        """
+        args = []
+        unnamed = {}
+        for f in fields:
+            if f.name is None:
+                name = f.type
+                c = unnamed[name] = unnamed.get(name, 0) + 1
+                if c > 1:
+                    name = "name%d" % (c - 1)
+            else:
+                name = f.name
+            # XXX should extend get_c_type() to handle this
+            if f.seq:
+                ctype = "asdl_seq *"
+            else:
+                ctype = get_c_type(f.type)
+            args.append((ctype, name, f.opt or f.seq))
+        return args
+
+    def visitConstructor(self, cons, type, attrs):
+        args = self.get_args(cons.fields)
+        attrs = self.get_args(attrs)
+        ctype = get_c_type(type)
+        self.emit_function(cons.name, ctype, args, attrs)
+
+    def emit_function(self, name, ctype, args, attrs, union=1):
+        args = args + attrs
+        if args:
+            argstr = ", ".join(["%s %s" % (atype, aname)
+                                for atype, aname, opt in args])
+        else:
+            argstr = "void"
+        self.emit("%s %s(%s);" % (ctype, name, argstr), 0)
+
+    def visitProduct(self, prod, name):
+        self.emit_function(name, get_c_type(name),
+                           self.get_args(prod.fields), [], union=0)
+
+class FunctionVisitor(PrototypeVisitor):
+    """Visitor to generate constructor functions for AST."""
+
+    def emit_function(self, name, ctype, args, attrs, union=1):
+        def emit(s, depth=0, reflow=1):
+            self.emit(s, depth, reflow)
+        argstr = ", ".join(["%s %s" % (atype, aname)
+                            for atype, aname, opt in args + attrs])
+        self.emit("%s" % ctype, 0)
+        emit("%s(%s)" % (name, argstr))
+        emit("{")
+        emit("%s p;" % ctype, 1)
+        for argtype, argname, opt in args:
+            # XXX hack alert: false is allowed for a bool
+            if not opt and not argtype == "bool":
+                emit("if (!%s) {" % argname, 1)
+                emit("PyErr_SetString(PyExc_ValueError,", 2)
+                msg = "field %s is required for %s" % (argname, name)
+                emit('                "%s");' % msg,
+                     2, reflow=0)
+                emit('return NULL;', 2)
+                emit('}', 1)
+
+        emit("p = (%s)malloc(sizeof(*p));" % ctype, 1)
+        emit("if (!p) {", 1)
+        emit("PyErr_SetString(PyExc_MemoryError, \"no memory\");", 2)
+        emit("return NULL;", 2)
+        emit("}", 1)
+        if union:
+            self.emit_body_union(name, args, attrs)
+        else:
+            self.emit_body_struct(name, args, attrs)
+        emit("return p;", 1)
+        emit("}")
+        emit("")
+
+    def emit_body_union(self, name, args, attrs):
+        def emit(s, depth=0, reflow=1):
+            self.emit(s, depth, reflow)
+        emit("p->kind = %s_kind;" % name, 1)
+        for argtype, argname, opt in args:
+            emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
+        for argtype, argname, opt in attrs:
+            emit("p->%s = %s;" % (argname, argname), 1)
+
+    def emit_body_struct(self, name, args, attrs):
+        def emit(s, depth=0, reflow=1):
+            self.emit(s, depth, reflow)
+        for argtype, argname, opt in args:
+            emit("p->%s = %s;" % (argname, argname), 1)
+        assert not attrs
+
+class PickleVisitor(EmitVisitor):
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type):
+        self.visit(type.value, type.name)
+
+    def visitSum(self, sum, name):
+        pass
+
+    def visitProduct(self, sum, name):
+        pass
+
+    def visitConstructor(self, cons, name):
+        pass
+
+    def visitField(self, sum):
+        pass
+
+class MarshalPrototypeVisitor(PickleVisitor):
+
+    def prototype(self, sum, name):
+        ctype = get_c_type(name)
+        self.emit("int marshal_write_%s(PyObject **, int *, %s);"
+                  % (name, ctype), 0)
+
+    visitProduct = visitSum = prototype
+
+class FreePrototypeVisitor(PickleVisitor):
+
+    def prototype(self, sum, name):
+        ctype = get_c_type(name)
+        self.emit("void free_%s(%s);" % (name, ctype), 0)
+
+    visitProduct = visitSum = prototype
+
+_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
+
+def find_sequence(fields, doing_specialization):
+    """Return True if any field uses a sequence."""
+    for f in fields:
+        if f.seq:
+            if not doing_specialization:
+                return True
+            if str(f.type) not in _SPECIALIZED_SEQUENCES:
+                return True
+    return False
+
+def has_sequence(types, doing_specialization):
+    for t in types:
+        if find_sequence(t.fields, doing_specialization):
+            return True
+    return False
+
+
+class StaticVisitor(PickleVisitor):
+    '''Very simple, always emit this static code'''
+
+    CODE = '''static void
+free_seq_exprs(asdl_seq *seq)
+{
+        int i, n;
+        n = asdl_seq_LEN(seq);
+        for (i = 0; i < n; i++)
+                free_expr((expr_ty)asdl_seq_GET(seq, i));
+        asdl_seq_free(seq);
+}
+
+static void
+free_seq_stmts(asdl_seq *seq)
+{
+        int i, n;
+        n = asdl_seq_LEN(seq);
+        for (i = 0; i < n; i++)
+                free_stmt((stmt_ty)asdl_seq_GET(seq, i));
+        asdl_seq_free(seq);
+}
+'''
+
+    def visit(self, object):
+        self.emit(self.CODE, 0, reflow=False)
+
+
+class FreeVisitor(PickleVisitor):
+
+    def func_begin(self, name, has_seq):
+        ctype = get_c_type(name)
+        self.emit("void", 0)
+        self.emit("free_%s(%s o)" % (name, ctype), 0)
+        self.emit("{", 0)
+        if has_seq:
+            self.emit("int i, n;", 1)
+            self.emit("asdl_seq *seq;", 1)
+            self.emit('', 0)
+        self.emit('if (!o)', 1)
+        self.emit('return;', 2)
+        self.emit('', 0)
+
+    def func_end(self):
+        self.emit("}", 0)
+        self.emit("", 0)
+
+    def visitSum(self, sum, name):
+        has_seq = has_sequence(sum.types, True)
+        self.func_begin(name, has_seq)
+        if not is_simple(sum):
+            self.emit("switch (o->kind) {", 1)
+            for i in range(len(sum.types)):
+                t = sum.types[i]
+                self.visitConstructor(t, i + 1, name)
+            self.emit("}", 1)
+            self.emit("", 0)
+            self.emit("free(o);", 1)
+        self.func_end()
+
+    def visitProduct(self, prod, name):
+        self.func_begin(name, find_sequence(prod.fields, True))
+        for field in prod.fields:
+            self.visitField(field, name, 1, True)
+        self.emit("", 0)
+        self.emit("free(o);", 1)
+        self.func_end()
+        
+    def visitConstructor(self, cons, enum, name):
+        self.emit("case %s_kind:" % cons.name, 1)
+        for f in cons.fields:
+            self.visitField(f, cons.name, 2, False)
+        self.emit("break;", 2)
+
+    def visitField(self, field, name, depth, product):
+        def emit(s, d):
+            self.emit(s, depth + d)
+        if product:
+            value = "o->%s" % field.name
+        else:
+            value = "o->v.%s.%s" % (name, field.name)
+        if field.seq:
+            self.emitSeq(field, value, depth, emit)
+
+        # XXX need to know the simple types in advance, so that we
+        # don't call free_TYPE() for them.
+
+        elif field.opt:
+            emit("if (%s) {" % value, 0)
+            self.free(field, value, depth + 1)
+            emit("}", 0)
+        else:
+            self.free(field, value, depth)
+
+    def emitSeq(self, field, value, depth, emit):
+        # specialize for freeing sequences of statements and expressions
+        if str(field.type) in _SPECIALIZED_SEQUENCES:
+            c_code = "free_seq_%ss(%s);" % (field.type, value)
+            emit(c_code, 0)
+        else:
+            emit("seq = %s;" % value, 0)
+            emit("n = asdl_seq_LEN(seq);", 0)
+            emit("for (i = 0; i < n; i++)", 0)
+            self.free(field, "asdl_seq_GET(seq, i)", depth + 1)
+            emit("asdl_seq_free(seq);", 0)
+
+    def free(self, field, value, depth):
+        if str(field.type) in ("identifier", "string", "object"):
+            ctype = get_c_type(field.type)
+            self.emit("Py_DECREF((%s)%s);" % (ctype, value), depth)
+        elif str(field.type) == "bool":
+            return
+        else:
+            ctype = get_c_type(field.type)
+            self.emit("free_%s((%s)%s);" % (field.type, ctype, value), depth)
+        
+
+class MarshalFunctionVisitor(PickleVisitor):
+
+    def func_begin(self, name, has_seq):
+        ctype = get_c_type(name)
+        self.emit("int", 0)
+        self.emit("marshal_write_%s(PyObject **buf, int *off, %s o)" %
+                  (name, ctype), 0)
+        self.emit("{", 0)
+        # XXX: add declaration of "int i;" properly
+        if has_seq or True:
+            self.emit("int i;", 1) # XXX only need it for sequences
+
+    def func_end(self):
+        self.emit("return 1;", 1)
+        self.emit("}", 0)
+        self.emit("", 0)
+    
+    def visitSum(self, sum, name):
+        has_seq = has_sequence(sum.types, False)
+        self.func_begin(name, has_seq)
+        simple = is_simple(sum)
+        if simple:
+            self.emit("switch (o) {", 1)
+        else:
+            self.emit("switch (o->kind) {", 1)
+        for i in range(len(sum.types)):
+            t = sum.types[i]
+            self.visitConstructor(t, i + 1, name, simple)
+        self.emit("}", 1)
+        self.func_end()
+
+    def visitProduct(self, prod, name):
+        self.func_begin(name, find_sequence(prod.fields, True))
+        for field in prod.fields:
+            self.visitField(field, name, 1, 1)
+        self.func_end()
+            
+    def visitConstructor(self, cons, enum, name, simple):
+        if simple:
+            self.emit("case %s:" % cons.name, 1)
+            self.emit("marshal_write_int(buf, off, %d);" % enum, 2);
+            self.emit("break;", 2)
+        else:
+            self.emit("case %s_kind:" % cons.name, 1)
+            self.emit("marshal_write_int(buf, off, %d);" % enum, 2)
+            for f in cons.fields:
+                self.visitField(f, cons.name, 2, 0)
+            self.emit("break;", 2)
+
+    def visitField(self, field, name, depth, product):
+        def emit(s, d):
+            self.emit(s, depth + d)
+        if product:
+            value = "o->%s" % field.name
+        else:
+            value = "o->v.%s.%s" % (name, field.name)
+        if field.seq:
+            emit("marshal_write_int(buf, off, asdl_seq_LEN(%s));" % value, 0)
+            emit("for (i = 0; i < asdl_seq_LEN(%s); i++) {" % value, 0)
+            emit("void *elt = asdl_seq_GET(%s, i);" % value, 1);
+            ctype = get_c_type(field.type);
+            emit("marshal_write_%s(buf, off, (%s)elt);" % (field.type,
+                    ctype), 1)
+            emit("}", 0)
+        elif field.opt:
+            emit("if (%s) {" % value, 0)
+            emit("marshal_write_int(buf, off, 1);", 1)
+            emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 1)
+            emit("}", 0)
+            emit("else {", 0)
+            emit("marshal_write_int(buf, off, 0);", 1)
+            emit("}", 0)
+        else:
+            emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 0)
+
+class ChainOfVisitors:
+    def __init__(self, *visitors):
+        self.visitors = visitors
+
+    def visit(self, object):
+        for v in self.visitors:
+            v.visit(object)
+
+def main(srcfile):
+    auto_gen_msg = '/* File automatically generated by %s */\n' % sys.argv[0]
+    mod = asdl.parse(srcfile)
+    if not asdl.check(mod):
+        sys.exit(1)
+    if INC_DIR:
+        p = "%s/%s-ast.h" % (INC_DIR, mod.name)
+    else:
+        p = "%s-ast.h" % mod.name
+    f = open(p, "wb")
+    print >> f, auto_gen_msg
+    print >> f, '#include "asdl.h"\n'
+    c = ChainOfVisitors(TypeDefVisitor(f),
+                        StructVisitor(f),
+                        PrototypeVisitor(f),
+                        FreePrototypeVisitor(f),
+                        MarshalPrototypeVisitor(f),
+                        )
+    c.visit(mod)
+    f.close()
+
+    if SRC_DIR:
+        p = "%s/%s-ast.c" % (SRC_DIR, mod.name)
+    else:
+        p = "%s-ast.c" % mod.name
+    f = open(p, "wb")
+    print >> f, auto_gen_msg
+    print >> f, '#include "Python.h"'
+    print >> f, '#include "%s-ast.h"' % mod.name
+    print >> f
+    v = ChainOfVisitors(FunctionVisitor(f),
+                        StaticVisitor(f),
+                        FreeVisitor(f),
+                        MarshalFunctionVisitor(f),
+                        )
+    v.visit(mod)
+    f.close()
+
+if __name__ == "__main__":
+    import sys
+    import getopt
+
+    INC_DIR = ''
+    SRC_DIR = ''
+    opts, args = getopt.getopt(sys.argv[1:], "h:c:")
+    for o, v in opts:
+        if o == '-h':
+            INC_DIR = v
+        if o == '-c':
+            SRC_DIR = v
+    if len(args) != 1:
+        print "Must specify single input file"
+    main(args[0])

Parser/grammar.mak

@@ -15,7 +15,7 @@
 # particular case --pragma in PC\pyconfig.h, which demands that
 # python23.lib get linked in).
 
-LIBS= ..\PCbuild\python23.lib
+LIBS= ..\PCbuild\python25.lib
 
 CFLAGS= /I ..\Include /I ..\PC /D MS_NO_COREDLL /D PGEN /MD
 

Parser/parsetok.c

@@ -21,7 +21,7 @@ static void initerr(perrdetail *err_ret, const char* filename);
 node *
 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
 {
-	return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
+	return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
 }
 
 node *
@@ -56,7 +56,6 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
 	return parsetok(tok, g, start, err_ret, flags);
 }
 
-
 /* Parse input coming from a file.  Return error code, print some errors. */
 
 node *
@@ -210,7 +209,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 }
 
 static void
-initerr(perrdetail *err_ret, const char* filename)
+initerr(perrdetail *err_ret, const char *filename)
 {
 	err_ret->error = E_OK;
 	err_ret->filename = filename;

Parser/spark.py

@@ -0,0 +1,840 @@
+#  Copyright (c) 1998-2002 John Aycock
+#  
+#  Permission is hereby granted, free of charge, to any person obtaining
+#  a copy of this software and associated documentation files (the
+#  "Software"), to deal in the Software without restriction, including
+#  without limitation the rights to use, copy, modify, merge, publish,
+#  distribute, sublicense, and/or sell copies of the Software, and to
+#  permit persons to whom the Software is furnished to do so, subject to
+#  the following conditions:
+#  
+#  The above copyright notice and this permission notice shall be
+#  included in all copies or substantial portions of the Software.
+#  
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+__version__ = 'SPARK-0.7 (pre-alpha-5)'
+
+import re
+import sys
+import string
+
+def _namelist(instance):
+	namelist, namedict, classlist = [], {}, [instance.__class__]
+	for c in classlist:
+		for b in c.__bases__:
+			classlist.append(b)
+		for name in c.__dict__.keys():
+			if not namedict.has_key(name):
+				namelist.append(name)
+				namedict[name] = 1
+	return namelist
+
+class GenericScanner:
+	def __init__(self, flags=0):
+		pattern = self.reflect()
+		self.re = re.compile(pattern, re.VERBOSE|flags)
+
+		self.index2func = {}
+		for name, number in self.re.groupindex.items():
+			self.index2func[number-1] = getattr(self, 't_' + name)
+
+	def makeRE(self, name):
+		doc = getattr(self, name).__doc__
+		rv = '(?P<%s>%s)' % (name[2:], doc)
+		return rv
+
+	def reflect(self):
+		rv = []
+		for name in _namelist(self):
+			if name[:2] == 't_' and name != 't_default':
+				rv.append(self.makeRE(name))
+
+		rv.append(self.makeRE('t_default'))
+		return string.join(rv, '|')
+
+	def error(self, s, pos):
+		print "Lexical error at position %s" % pos
+		raise SystemExit
+
+	def tokenize(self, s):
+		pos = 0
+		n = len(s)
+		while pos < n:
+			m = self.re.match(s, pos)
+			if m is None:
+				self.error(s, pos)
+
+			groups = m.groups()
+			for i in range(len(groups)):
+				if groups[i] and self.index2func.has_key(i):
+					self.index2func[i](groups[i])
+			pos = m.end()
+
+	def t_default(self, s):
+		r'( . | \n )+'
+		print "Specification error: unmatched input"
+		raise SystemExit
+
+#
+#  Extracted from GenericParser and made global so that [un]picking works.
+#
+class _State:
+	def __init__(self, stateno, items):
+		self.T, self.complete, self.items = [], [], items
+		self.stateno = stateno
+
+class GenericParser:
+	#
+	#  An Earley parser, as per J. Earley, "An Efficient Context-Free
+	#  Parsing Algorithm", CACM 13(2), pp. 94-102.  Also J. C. Earley,
+	#  "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
+	#  Carnegie-Mellon University, August 1968.  New formulation of
+	#  the parser according to J. Aycock, "Practical Earley Parsing
+	#  and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
+	#  2001, and J. Aycock and R. N. Horspool, "Practical Earley
+	#  Parsing", unpublished paper, 2001.
+	#
+
+	def __init__(self, start):
+		self.rules = {}
+		self.rule2func = {}
+		self.rule2name = {}
+		self.collectRules()
+		self.augment(start)
+		self.ruleschanged = 1
+
+	_NULLABLE = '\e_'
+	_START = 'START'
+	_BOF = '|-'
+
+	#
+	#  When pickling, take the time to generate the full state machine;
+	#  some information is then extraneous, too.  Unfortunately we
+	#  can't save the rule2func map.
+	#
+	def __getstate__(self):
+		if self.ruleschanged:
+			#
+			#  XXX - duplicated from parse()
+			#
+			self.computeNull()
+			self.newrules = {}
+			self.new2old = {}
+			self.makeNewRules()
+			self.ruleschanged = 0
+			self.edges, self.cores = {}, {}
+			self.states = { 0: self.makeState0() }
+			self.makeState(0, self._BOF)
+		#
+		#  XXX - should find a better way to do this..
+		#
+		changes = 1
+		while changes:
+			changes = 0
+			for k, v in self.edges.items():
+				if v is None:
+					state, sym = k
+					if self.states.has_key(state):
+						self.goto(state, sym)
+						changes = 1
+		rv = self.__dict__.copy()
+		for s in self.states.values():
+			del s.items
+		del rv['rule2func']
+		del rv['nullable']
+		del rv['cores']
+		return rv
+
+	def __setstate__(self, D):
+		self.rules = {}
+		self.rule2func = {}
+		self.rule2name = {}
+		self.collectRules()
+		start = D['rules'][self._START][0][1][1]	# Blech.
+		self.augment(start)
+		D['rule2func'] = self.rule2func
+		D['makeSet'] = self.makeSet_fast
+		self.__dict__ = D
+
+	#
+	#  A hook for GenericASTBuilder and GenericASTMatcher.  Mess
+	#  thee not with this; nor shall thee toucheth the _preprocess
+	#  argument to addRule.
+	#
+	def preprocess(self, rule, func):	return rule, func
+
+	def addRule(self, doc, func, _preprocess=1):
+		fn = func
+		rules = string.split(doc)
+
+		index = []
+		for i in range(len(rules)):
+			if rules[i] == '::=':
+				index.append(i-1)
+		index.append(len(rules))
+
+		for i in range(len(index)-1):
+			lhs = rules[index[i]]
+			rhs = rules[index[i]+2:index[i+1]]
+			rule = (lhs, tuple(rhs))
+
+			if _preprocess:
+				rule, fn = self.preprocess(rule, func)
+
+			if self.rules.has_key(lhs):
+				self.rules[lhs].append(rule)
+			else:
+				self.rules[lhs] = [ rule ]
+			self.rule2func[rule] = fn
+			self.rule2name[rule] = func.__name__[2:]
+		self.ruleschanged = 1
+
+	def collectRules(self):
+		for name in _namelist(self):
+			if name[:2] == 'p_':
+				func = getattr(self, name)
+				doc = func.__doc__
+				self.addRule(doc, func)
+
+	def augment(self, start):
+		rule = '%s ::= %s %s' % (self._START, self._BOF, start)
+		self.addRule(rule, lambda args: args[1], 0)
+
+	def computeNull(self):
+		self.nullable = {}
+		tbd = []
+
+		for rulelist in self.rules.values():
+			lhs = rulelist[0][0]
+			self.nullable[lhs] = 0
+			for rule in rulelist:
+				rhs = rule[1]
+				if len(rhs) == 0:
+					self.nullable[lhs] = 1
+					continue
+				#
+				#  We only need to consider rules which
+				#  consist entirely of nonterminal symbols.
+				#  This should be a savings on typical
+				#  grammars.
+				#
+				for sym in rhs:
+					if not self.rules.has_key(sym):
+						break
+				else:
+					tbd.append(rule)
+		changes = 1
+		while changes:
+			changes = 0
+			for lhs, rhs in tbd:
+				if self.nullable[lhs]:
+					continue
+				for sym in rhs:
+					if not self.nullable[sym]:
+						break
+				else:
+					self.nullable[lhs] = 1
+					changes = 1
+
+	def makeState0(self):
+		s0 = _State(0, [])
+		for rule in self.newrules[self._START]:
+			s0.items.append((rule, 0))
+		return s0
+
+	def finalState(self, tokens):
+		#
+		#  Yuck.
+		#
+		if len(self.newrules[self._START]) == 2 and len(tokens) == 0:
+			return 1
+		start = self.rules[self._START][0][1][1]
+		return self.goto(1, start)
+
+	def makeNewRules(self):
+		worklist = []
+		for rulelist in self.rules.values():
+			for rule in rulelist:
+				worklist.append((rule, 0, 1, rule))
+
+		for rule, i, candidate, oldrule in worklist:
+			lhs, rhs = rule
+			n = len(rhs)
+			while i < n:
+				sym = rhs[i]
+				if not self.rules.has_key(sym) or \
+				   not self.nullable[sym]:
+					candidate = 0
+					i = i + 1
+					continue
+
+				newrhs = list(rhs)
+				newrhs[i] = self._NULLABLE+sym
+				newrule = (lhs, tuple(newrhs))
+				worklist.append((newrule, i+1,
+						 candidate, oldrule))
+				candidate = 0
+				i = i + 1
+			else:
+				if candidate:
+					lhs = self._NULLABLE+lhs
+					rule = (lhs, rhs)
+				if self.newrules.has_key(lhs):
+					self.newrules[lhs].append(rule)
+				else:
+					self.newrules[lhs] = [ rule ]
+				self.new2old[rule] = oldrule
+	
+	def typestring(self, token):
+		return None
+
+	def error(self, token):
+		print "Syntax error at or near `%s' token" % token
+		raise SystemExit
+
+	def parse(self, tokens):
+		sets = [ [(1,0), (2,0)] ]
+		self.links = {}
+		
+		if self.ruleschanged:
+			self.computeNull()
+			self.newrules = {}
+			self.new2old = {}
+			self.makeNewRules()
+			self.ruleschanged = 0
+			self.edges, self.cores = {}, {}
+			self.states = { 0: self.makeState0() }
+			self.makeState(0, self._BOF)
+
+		for i in xrange(len(tokens)):
+			sets.append([])
+
+			if sets[i] == []:
+				break				
+			self.makeSet(tokens[i], sets, i)
+		else:
+			sets.append([])
+			self.makeSet(None, sets, len(tokens))
+
+		#_dump(tokens, sets, self.states)
+
+		finalitem = (self.finalState(tokens), 0)
+		if finalitem not in sets[-2]:
+			if len(tokens) > 0:
+				self.error(tokens[i-1])
+			else:
+				self.error(None)
+
+		return self.buildTree(self._START, finalitem,
+				      tokens, len(sets)-2)
+
+	def isnullable(self, sym):
+		#
+		#  For symbols in G_e only.  If we weren't supporting 1.5,
+		#  could just use sym.startswith().
+		#
+		return self._NULLABLE == sym[0:len(self._NULLABLE)]
+
+	def skip(self, (lhs, rhs), pos=0):
+		n = len(rhs)
+		while pos < n:
+			if not self.isnullable(rhs[pos]):
+				break
+			pos = pos + 1
+		return pos
+
+	def makeState(self, state, sym):
+		assert sym is not None
+		#
+		#  Compute \epsilon-kernel state's core and see if
+		#  it exists already.
+		#
+		kitems = []
+		for rule, pos in self.states[state].items:
+			lhs, rhs = rule
+			if rhs[pos:pos+1] == (sym,):
+				kitems.append((rule, self.skip(rule, pos+1)))
+		core = kitems
+
+		core.sort()
+		tcore = tuple(core)
+		if self.cores.has_key(tcore):
+			return self.cores[tcore]
+		#
+		#  Nope, doesn't exist.  Compute it and the associated
+		#  \epsilon-nonkernel state together; we'll need it right away.
+		#
+		k = self.cores[tcore] = len(self.states)
+		K, NK = _State(k, kitems), _State(k+1, [])
+		self.states[k] = K
+		predicted = {}
+
+		edges = self.edges
+		rules = self.newrules
+		for X in K, NK:
+			worklist = X.items
+			for item in worklist:
+				rule, pos = item
+				lhs, rhs = rule
+				if pos == len(rhs):
+					X.complete.append(rule)
+					continue
+
+				nextSym = rhs[pos]
+				key = (X.stateno, nextSym)
+				if not rules.has_key(nextSym):
+					if not edges.has_key(key):
+						edges[key] = None
+						X.T.append(nextSym)
+				else:
+					edges[key] = None
+					if not predicted.has_key(nextSym):
+						predicted[nextSym] = 1
+						for prule in rules[nextSym]:
+							ppos = self.skip(prule)
+							new = (prule, ppos)
+							NK.items.append(new)
+			#
+			#  Problem: we know K needs generating, but we
+			#  don't yet know about NK.  Can't commit anything
+			#  regarding NK to self.edges until we're sure.  Should
+			#  we delay committing on both K and NK to avoid this
+			#  hacky code?  This creates other problems..
+			#
+			if X is K:
+				edges = {}
+
+		if NK.items == []:
+			return k
+
+		#
+		#  Check for \epsilon-nonkernel's core.  Unfortunately we
+		#  need to know the entire set of predicted nonterminals
+		#  to do this without accidentally duplicating states.
+		#
+		core = predicted.keys()
+		core.sort()
+		tcore = tuple(core)
+		if self.cores.has_key(tcore):
+			self.edges[(k, None)] = self.cores[tcore]
+			return k
+
+		nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno
+		self.edges.update(edges)
+		self.states[nk] = NK
+		return k
+
+	def goto(self, state, sym):
+		key = (state, sym)
+		if not self.edges.has_key(key):
+			#
+			#  No transitions from state on sym.
+			#
+			return None
+
+		rv = self.edges[key]
+		if rv is None:
+			#
+			#  Target state isn't generated yet.  Remedy this.
+			#
+			rv = self.makeState(state, sym)
+			self.edges[key] = rv
+		return rv
+
+	def gotoT(self, state, t):
+		return [self.goto(state, t)]
+
+	def gotoST(self, state, st):
+		rv = []
+		for t in self.states[state].T:
+			if st == t:
+				rv.append(self.goto(state, t))
+		return rv
+
+	def add(self, set, item, i=None, predecessor=None, causal=None):
+		if predecessor is None:
+			if item not in set:
+				set.append(item)
+		else:
+			key = (item, i)
+			if item not in set:
+				self.links[key] = []
+				set.append(item)
+			self.links[key].append((predecessor, causal))
+
+	def makeSet(self, token, sets, i):
+		cur, next = sets[i], sets[i+1]
+
+		ttype = token is not None and self.typestring(token) or None
+		if ttype is not None:
+			fn, arg = self.gotoT, ttype
+		else:
+			fn, arg = self.gotoST, token
+
+		for item in cur:
+			ptr = (item, i)
+			state, parent = item
+			add = fn(state, arg)
+			for k in add:
+				if k is not None:
+					self.add(next, (k, parent), i+1, ptr)
+					nk = self.goto(k, None)
+					if nk is not None:
+						self.add(next, (nk, i+1))
+
+			if parent == i:
+				continue
+
+			for rule in self.states[state].complete:
+				lhs, rhs = rule
+				for pitem in sets[parent]:
+					pstate, pparent = pitem
+					k = self.goto(pstate, lhs)
+					if k is not None:
+						why = (item, i, rule)
+						pptr = (pitem, parent)
+						self.add(cur, (k, pparent),
+							 i, pptr, why)
+						nk = self.goto(k, None)
+						if nk is not None:
+							self.add(cur, (nk, i))
+
+	def makeSet_fast(self, token, sets, i):
+		#
+		#  Call *only* when the entire state machine has been built!
+		#  It relies on self.edges being filled in completely, and
+		#  then duplicates and inlines code to boost speed at the
+		#  cost of extreme ugliness.
+		#
+		cur, next = sets[i], sets[i+1]
+		ttype = token is not None and self.typestring(token) or None
+
+		for item in cur:
+			ptr = (item, i)
+			state, parent = item
+			if ttype is not None:
+				k = self.edges.get((state, ttype), None)
+				if k is not None:
+					#self.add(next, (k, parent), i+1, ptr)
+					#INLINED --v
+					new = (k, parent)
+					key = (new, i+1)
+					if new not in next:
+						self.links[key] = []
+						next.append(new)
+					self.links[key].append((ptr, None))
+					#INLINED --^
+					#nk = self.goto(k, None)
+					nk = self.edges.get((k, None), None)
+					if nk is not None:
+						#self.add(next, (nk, i+1))
+						#INLINED --v
+						new = (nk, i+1)
+						if new not in next:
+							next.append(new)
+						#INLINED --^
+			else:
+				add = self.gotoST(state, token)
+				for k in add:
+					if k is not None:
+						self.add(next, (k, parent), i+1, ptr)
+						#nk = self.goto(k, None)
+						nk = self.edges.get((k, None), None)
+						if nk is not None:
+							self.add(next, (nk, i+1))
+
+			if parent == i:
+				continue
+
+			for rule in self.states[state].complete:
+				lhs, rhs = rule
+				for pitem in sets[parent]:
+					pstate, pparent = pitem
+					#k = self.goto(pstate, lhs)
+					k = self.edges.get((pstate, lhs), None)
+					if k is not None:
+						why = (item, i, rule)
+						pptr = (pitem, parent)
+						#self.add(cur, (k, pparent),
+						#	 i, pptr, why)
+						#INLINED --v
+						new = (k, pparent)
+						key = (new, i)
+						if new not in cur:
+							self.links[key] = []
+							cur.append(new)
+						self.links[key].append((pptr, why))
+						#INLINED --^
+						#nk = self.goto(k, None)
+						nk = self.edges.get((k, None), None)
+						if nk is not None:
+							#self.add(cur, (nk, i))
+							#INLINED --v
+							new = (nk, i)
+							if new not in cur:
+								cur.append(new)
+							#INLINED --^
+
+	def predecessor(self, key, causal):
+		for p, c in self.links[key]:
+			if c == causal:
+				return p
+		assert 0
+
+	def causal(self, key):
+		links = self.links[key]
+		if len(links) == 1:
+			return links[0][1]
+		choices = []
+		rule2cause = {}
+		for p, c in links:
+			rule = c[2]
+			choices.append(rule)
+			rule2cause[rule] = c
+		return rule2cause[self.ambiguity(choices)]
+
+	def deriveEpsilon(self, nt):
+		if len(self.newrules[nt]) > 1:
+			rule = self.ambiguity(self.newrules[nt])
+		else:
+			rule = self.newrules[nt][0]
+		#print rule
+
+		rhs = rule[1]
+		attr = [None] * len(rhs)
+
+		for i in range(len(rhs)-1, -1, -1):
+			attr[i] = self.deriveEpsilon(rhs[i])
+		return self.rule2func[self.new2old[rule]](attr)
+
+	def buildTree(self, nt, item, tokens, k):
+		state, parent = item
+
+		choices = []
+		for rule in self.states[state].complete:
+			if rule[0] == nt:
+				choices.append(rule)
+		rule = choices[0]
+		if len(choices) > 1:
+			rule = self.ambiguity(choices)
+		#print rule
+
+		rhs = rule[1]
+		attr = [None] * len(rhs)
+
+		for i in range(len(rhs)-1, -1, -1):
+			sym = rhs[i]
+			if not self.newrules.has_key(sym):
+				if sym != self._BOF:
+					attr[i] = tokens[k-1]
+					key = (item, k)
+					item, k = self.predecessor(key, None)
+			#elif self.isnullable(sym):
+			elif self._NULLABLE == sym[0:len(self._NULLABLE)]:
+				attr[i] = self.deriveEpsilon(sym)
+			else:
+				key = (item, k)
+				why = self.causal(key)
+				attr[i] = self.buildTree(sym, why[0],
+							 tokens, why[1])
+				item, k = self.predecessor(key, why)
+		return self.rule2func[self.new2old[rule]](attr)
+
+	def ambiguity(self, rules):
+		#
+		#  XXX - problem here and in collectRules() if the same rule
+		#	 appears in >1 method.  Also undefined results if rules
+		#	 causing the ambiguity appear in the same method.
+		#
+		sortlist = []
+		name2index = {}
+		for i in range(len(rules)):
+			lhs, rhs = rule = rules[i]
+			name = self.rule2name[self.new2old[rule]]
+			sortlist.append((len(rhs), name))
+			name2index[name] = i
+		sortlist.sort()
+		list = map(lambda (a,b): b, sortlist)
+		return rules[name2index[self.resolve(list)]]
+
+	def resolve(self, list):
+		#
+		#  Resolve ambiguity in favor of the shortest RHS.
+		#  Since we walk the tree from the top down, this
+		#  should effectively resolve in favor of a "shift".
+		#
+		return list[0]
+
+#
+#  GenericASTBuilder automagically constructs a concrete/abstract syntax tree
+#  for a given input.  The extra argument is a class (not an instance!)
+#  which supports the "__setslice__" and "__len__" methods.
+#
+#  XXX - silently overrides any user code in methods.
+#
+
+class GenericASTBuilder(GenericParser):
+	def __init__(self, AST, start):
+		GenericParser.__init__(self, start)
+		self.AST = AST
+
+	def preprocess(self, rule, func):
+		rebind = lambda lhs, self=self: \
+				lambda args, lhs=lhs, self=self: \
+					self.buildASTNode(args, lhs)
+		lhs, rhs = rule
+		return rule, rebind(lhs)
+
+	def buildASTNode(self, args, lhs):
+		children = []
+		for arg in args:
+			if isinstance(arg, self.AST):
+				children.append(arg)
+			else:
+				children.append(self.terminal(arg))
+		return self.nonterminal(lhs, children)
+
+	def terminal(self, token):	return token
+
+	def nonterminal(self, type, args):
+		rv = self.AST(type)
+		rv[:len(args)] = args
+		return rv
+
+#
+#  GenericASTTraversal is a Visitor pattern according to Design Patterns.  For
+#  each node it attempts to invoke the method n_<node type>, falling
+#  back onto the default() method if the n_* can't be found.  The preorder
+#  traversal also looks for an exit hook named n_<node type>_exit (no default
+#  routine is called if it's not found).  To prematurely halt traversal
+#  of a subtree, call the prune() method -- this only makes sense for a
+#  preorder traversal.  Node type is determined via the typestring() method.
+#
+
+class GenericASTTraversalPruningException:
+	pass
+
+class GenericASTTraversal:
+	def __init__(self, ast):
+		self.ast = ast
+
+	def typestring(self, node):
+		return node.type
+
+	def prune(self):
+		raise GenericASTTraversalPruningException
+
+	def preorder(self, node=None):
+		if node is None:
+			node = self.ast
+
+		try:
+			name = 'n_' + self.typestring(node)
+			if hasattr(self, name):
+				func = getattr(self, name)
+				func(node)
+			else:
+				self.default(node)
+		except GenericASTTraversalPruningException:
+			return
+
+		for kid in node:
+			self.preorder(kid)
+
+		name = name + '_exit'
+		if hasattr(self, name):
+			func = getattr(self, name)
+			func(node)
+
+	def postorder(self, node=None):
+		if node is None:
+			node = self.ast
+
+		for kid in node:
+			self.postorder(kid)
+
+		name = 'n_' + self.typestring(node)
+		if hasattr(self, name):
+			func = getattr(self, name)
+			func(node)
+		else:
+			self.default(node)
+
+
+	def default(self, node):
+		pass
+
+#
+#  GenericASTMatcher.  AST nodes must have "__getitem__" and "__cmp__"
+#  implemented.
+#
+#  XXX - makes assumptions about how GenericParser walks the parse tree.
+#
+
+class GenericASTMatcher(GenericParser):
+	def __init__(self, start, ast):
+		GenericParser.__init__(self, start)
+		self.ast = ast
+
+	def preprocess(self, rule, func):
+		rebind = lambda func, self=self: \
+				lambda args, func=func, self=self: \
+					self.foundMatch(args, func)
+		lhs, rhs = rule
+		rhslist = list(rhs)
+		rhslist.reverse()
+
+		return (lhs, tuple(rhslist)), rebind(func)
+
+	def foundMatch(self, args, func):
+		func(args[-1])
+		return args[-1]
+
+	def match_r(self, node):
+		self.input.insert(0, node)
+		children = 0
+
+		for child in node:
+			if children == 0:
+				self.input.insert(0, '(')
+			children = children + 1
+			self.match_r(child)
+
+		if children > 0:
+			self.input.insert(0, ')')
+
+	def match(self, ast=None):
+		if ast is None:
+			ast = self.ast
+		self.input = []
+
+		self.match_r(ast)
+		self.parse(self.input)
+
+	def resolve(self, list):
+		#
+		#  Resolve ambiguity in favor of the longest RHS.
+		#
+		return list[-1]
+
+def _dump(tokens, sets, states):
+	for i in range(len(sets)):
+		print 'set', i
+		for item in sets[i]:
+			print '\t', item
+			for (lhs, rhs), pos in states[item[0]].items:
+				print '\t\t', lhs, '::=',
+				print string.join(rhs[:pos]),
+				print '.',
+				print string.join(rhs[pos:])
+		if i < len(tokens):
+			print
+			print 'token', str(tokens[i])
+			print

Python/asdl.c

@@ -0,0 +1,92 @@
+#include "Python.h"
+#include "asdl.h"
+
+asdl_seq *
+asdl_seq_new(int size)
+{
+	asdl_seq *seq = NULL;
+	size_t n = sizeof(asdl_seq) +
+			(size ? (sizeof(void *) * (size - 1)) : 0);
+
+	seq = (asdl_seq *)PyObject_Malloc(n);
+	if (!seq) {
+		PyErr_SetString(PyExc_MemoryError, "no memory");
+		return NULL;
+	}
+	memset(seq, 0, n);
+	seq->size = size;
+	return seq;
+}
+
+void
+asdl_seq_free(asdl_seq *seq)
+{
+	PyObject_Free(seq);
+}
+
+#define CHECKSIZE(BUF, OFF, MIN) { \
+	int need = *(OFF) + MIN; \
+	if (need >= PyString_GET_SIZE(*(BUF))) { \
+		int newsize = PyString_GET_SIZE(*(BUF)) * 2; \
+		if (newsize < need) \
+			newsize = need; \
+		if (_PyString_Resize((BUF), newsize) < 0) \
+			return 0; \
+	} \
+} 
+
+int 
+marshal_write_int(PyObject **buf, int *offset, int x)
+{
+	char *s;
+
+	CHECKSIZE(buf, offset, 4)
+	s = PyString_AS_STRING(*buf) + (*offset);
+	s[0] = (x & 0xff);
+	s[1] = (x >> 8) & 0xff;
+	s[2] = (x >> 16) & 0xff;
+	s[3] = (x >> 24) & 0xff;
+	*offset += 4;
+	return 1;
+}
+
+int 
+marshal_write_bool(PyObject **buf, int *offset, bool b)
+{
+	if (b)
+		marshal_write_int(buf, offset, 1);
+	else
+		marshal_write_int(buf, offset, 0);
+	return 1;
+}
+
+int 
+marshal_write_identifier(PyObject **buf, int *offset, identifier id)
+{
+	int l = PyString_GET_SIZE(id);
+	marshal_write_int(buf, offset, l);
+	CHECKSIZE(buf, offset, l);
+	memcpy(PyString_AS_STRING(*buf) + *offset,
+	       PyString_AS_STRING(id), l);
+	*offset += l;
+	return 1;
+}
+
+int 
+marshal_write_string(PyObject **buf, int *offset, string s)
+{
+	int len = PyString_GET_SIZE(s);
+	marshal_write_int(buf, offset, len);
+	CHECKSIZE(buf, offset, len);
+	memcpy(PyString_AS_STRING(*buf) + *offset,
+	       PyString_AS_STRING(s), len);
+	*offset += len;
+	return 1;
+}
+
+int 
+marshal_write_object(PyObject **buf, int *offset, object s)
+{
+	/* XXX */
+	return 0;
+}

Python/bltinmodule.c

@@ -1,9 +1,9 @@
-
 /* Built-in functions */
 
 #include "Python.h"
 
 #include "node.h"
+#include "code.h"
 #include "compile.h"
 #include "eval.h"
 

Python/ceval.c

@@ -8,7 +8,7 @@
 
 #include "Python.h"
 
-#include "compile.h"
+#include "code.h"
 #include "frameobject.h"
 #include "eval.h"
 #include "opcode.h"
@@ -543,7 +543,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
 #ifdef LLTRACE
 	int lltrace;
 #endif
-#if defined(Py_DEBUG) || defined(LLTRACE)
+#if defined(Py_DEBUG)
 	/* Make it easier to find out where we are with a debugger */
 	char *filename;
 #endif
@@ -743,9 +743,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
 	f->f_stacktop = NULL;	/* remains NULL unless yield suspends frame */
 
 #ifdef LLTRACE
-	lltrace = PyDict_GetItemString(f->f_globals,"__lltrace__") != NULL;
+	lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
 #endif
-#if defined(Py_DEBUG) || defined(LLTRACE)
+#if defined(Py_DEBUG)
 	filename = PyString_AsString(co->co_filename);
 #endif
 
@@ -2257,23 +2257,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
 
 		case MAKE_CLOSURE:
 		{
-			int nfree;
 			v = POP(); /* code object */
 			x = PyFunction_New(v, f->f_globals);
-			nfree = PyCode_GetNumFree((PyCodeObject *)v);
 			Py_DECREF(v);
-			/* XXX Maybe this should be a separate opcode? */
+			if (x != NULL) {
-			if (x != NULL && nfree > 0) {
+				v = POP();
-				v = PyTuple_New(nfree);
-				if (v == NULL) {
-					Py_DECREF(x);
-					x = NULL;
-					break;
-				}
-				while (--nfree >= 0) {
-					w = POP();
-					PyTuple_SET_ITEM(v, nfree, w);
-				}
 				err = PyFunction_SetClosure(x, v);
 				Py_DECREF(v);
 			}
@@ -2695,12 +2683,18 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 		if (co->co_flags & CO_VARKEYWORDS)
 			nargs++;
 
-		/* Check for cells that shadow args */
+		/* Initialize each cell var, taking into account
-		for (i = 0; i < f->f_ncells && j < nargs; ++i) {
+		   cell vars that are initialized from arguments.
+
+		   Should arrange for the compiler to put cellvars
+		   that are arguments at the beginning of the cellvars
+		   list so that we can march over it more efficiently?
+		*/
+		for (i = 0; i < f->f_ncells; ++i) {
 			cellname = PyString_AS_STRING(
 				PyTuple_GET_ITEM(co->co_cellvars, i));
 			found = 0;
-			while (j < nargs) {
+			for (j = 0; j < nargs; j++) {
 				argname = PyString_AS_STRING(
 					PyTuple_GET_ITEM(co->co_varnames, j));
 				if (strcmp(cellname, argname) == 0) {
@@ -2711,7 +2705,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 					found = 1;
 					break;
 				}
-				j++;
 			}
 			if (found == 0) {
 				c = PyCell_New(NULL);
@@ -2720,14 +2713,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 				SETLOCAL(f->f_nlocals + i, c);
 			}
 		}
-		/* Initialize any that are left */
-		while (i < f->f_ncells) {
-			c = PyCell_New(NULL);
-			if (c == NULL)
-				goto fail;
-			SETLOCAL(f->f_nlocals + i, c);
-			i++;
-		}
 	}
 	if (f->f_nfreevars) {
 		int i;

Python/future.c

@@ -1,37 +1,30 @@
 #include "Python.h"
+#include "Python-ast.h"
 #include "node.h"
 #include "token.h"
 #include "graminit.h"
+#include "code.h"
 #include "compile.h"
 #include "symtable.h"
 
 #define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined"
 #define FUTURE_IMPORT_STAR "future statement does not support import *"
 
-/* FUTURE_POSSIBLE() is provided to accomodate doc strings, which is
-   the only statement that can occur before a future statement.
-*/
-#define FUTURE_POSSIBLE(FF) ((FF)->ff_last_lineno == -1)
-
 static int
-future_check_features(PyFutureFeatures *ff, node *n, const char *filename)
+future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
 {
 	int i;
-	char *feature;
+	const char *feature;
-	node *ch, *nn;
+	asdl_seq *names;
 
-	REQ(n, import_from);
+	assert(s->kind == ImportFrom_kind);
-	nn = CHILD(n, 3 + (TYPE(CHILD(n, 3)) == LPAR));
+
-	if (TYPE(nn) == STAR) {
+	names = s->v.ImportFrom.names;
-		PyErr_SetString(PyExc_SyntaxError, FUTURE_IMPORT_STAR);
+	for (i = 0; i < asdl_seq_LEN(names); i++) {
-		PyErr_SyntaxLocation(filename, nn->n_lineno);
+                alias_ty name = asdl_seq_GET(names, i);
-		return -1;
+		feature = PyString_AsString(name->name);
-	}
+		if (!feature)
-	REQ(nn, import_as_names);
+			return 0;
-	for (i = 0; i < NCH(nn); i += 2) {
-		ch = CHILD(nn, i);
-		REQ(ch, import_as_name);
-		feature = STR(CHILD(ch, 0));
 		if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
 			continue;
 		} else if (strcmp(feature, FUTURE_GENERATORS) == 0) {
@@ -41,218 +34,97 @@ future_check_features(PyFutureFeatures *ff, node *n, const char *filename)
 		} else if (strcmp(feature, "braces") == 0) {
 			PyErr_SetString(PyExc_SyntaxError,
 					"not a chance");
-			PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno);
+			PyErr_SyntaxLocation(filename, s->lineno);
-			return -1;
+			return 0;
 		} else {
 			PyErr_Format(PyExc_SyntaxError,
 				     UNDEFINED_FUTURE_FEATURE, feature);
-			PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno);
+			PyErr_SyntaxLocation(filename, s->lineno);
-			return -1;
+			return 0;
 		}
 	}
-	return 0;
+	return 1;
 }
 
-static void
+int
-future_error(node *n, const char *filename)
+future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename)
 {
-	PyErr_SetString(PyExc_SyntaxError,
+	int i, found_docstring = 0, done = 0, prev_line = 0;
-			"from __future__ imports must occur at the "
-			"beginning of the file");
-	PyErr_SyntaxLocation(filename, n->n_lineno);
-}
 
-/* Relevant portions of the grammar:
+	static PyObject *future;
-
+	if (!future) {
-single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+		future = PyString_InternFromString("__future__");
-file_input: (NEWLINE | stmt)* ENDMARKER
+		if (!future)
-stmt: simple_stmt | compound_stmt
-simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
-small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt 
-    | import_stmt | global_stmt | exec_stmt | assert_stmt
-import_stmt: 'import' dotted_as_name (',' dotted_as_name)* 
-    | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
-import_as_name: NAME [NAME NAME]
-dotted_as_name: dotted_name [NAME NAME]
-dotted_name: NAME ('.' NAME)*
-*/
-
-/* future_parse() finds future statements at the beginnning of a
-   module.  The function calls itself recursively, rather than
-   factoring out logic for different kinds of statements into
-   different routines.
-
-   Return values:
-   -1 indicates an error occurred, e.g. unknown feature name
-   0 indicates no feature was found
-   1 indicates a feature was found
-*/
-
-static int
-future_parse(PyFutureFeatures *ff, node *n, const char *filename)
-{
-	int i, r;
- loop:
-	switch (TYPE(n)) {
-
-	case single_input:
-		if (TYPE(CHILD(n, 0)) == simple_stmt) {
-			n = CHILD(n, 0);
-			goto loop;
-		}
-		return 0;
-
-	case file_input:
-		/* Check each statement in the file, starting with the
-		   first, and continuing until the first statement
-		   that isn't a future statement.
-		*/
-		for (i = 0; i < NCH(n); i++) {
-			node *ch = CHILD(n, i);
-			if (TYPE(ch) == stmt) {
-				r = future_parse(ff, ch, filename);
-				/* Need to check both conditions below
-				   to accomodate doc strings, which
-				   causes r < 0.
-				*/
-				if (r < 1 && !FUTURE_POSSIBLE(ff))
-					return r;
-			}
-		}
-		return 0;
-
-	case simple_stmt:
-		if (NCH(n) == 2) {
-			REQ(CHILD(n, 0), small_stmt);
-			n = CHILD(n, 0);
-			goto loop;
-		} else {
-			/* Deal with the special case of a series of
-			   small statements on a single line.  If a
-			   future statement follows some other
-			   statement, the SyntaxError is raised here.
-			   In all other cases, the symtable pass
-			   raises the exception.
-			*/
-			int found = 0, end_of_future = 0;
-
-			for (i = 0; i < NCH(n); i += 2) {
-				if (TYPE(CHILD(n, i)) == small_stmt) {
-					r = future_parse(ff, CHILD(n, i), 
-							 filename);
-					if (r < 1)
-						end_of_future = 1;
-					else {
-						found = 1;
-						if (end_of_future) {
-							future_error(n, 
-								     filename);
-							return -1;
-						}
-					}
-				}
-			}
-
-			/* If we found one and only one, then the
-			   current lineno is legal. 
-			*/
-			if (found)
-				ff->ff_last_lineno = n->n_lineno + 1;
-			else
-				ff->ff_last_lineno = n->n_lineno;
-
-			if (end_of_future && found)
-				return 1;
-			else 
-				return 0;
-		}
-	
-	case stmt:
-		if (TYPE(CHILD(n, 0)) == simple_stmt) {
-			n = CHILD(n, 0);
-			goto loop;
-		} else if (TYPE(CHILD(n, 0)) == expr_stmt) {
-			n = CHILD(n, 0);
-			goto loop;
-		} else {
-			REQ(CHILD(n, 0), compound_stmt);
-			ff->ff_last_lineno = n->n_lineno;
 			return 0;
-		}
+	}
 
-	case small_stmt:
+	if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
-		n = CHILD(n, 0);
-		goto loop;
-
-	case import_stmt: {
-		node *name;
-
-		n = CHILD(n, 0);
-		if (TYPE(n) != import_from) {
-			ff->ff_last_lineno = n->n_lineno;
-			return 0;
-		}
-		name = CHILD(n, 1);
-		if (strcmp(STR(CHILD(name, 0)), "__future__") != 0)
-			return 0;
-		if (future_check_features(ff, n, filename) < 0)
-			return -1;
-		ff->ff_last_lineno = n->n_lineno + 1;
 		return 1;
-	}
 
-	/* The cases below -- all of them! -- are necessary to find
+	/* A subsequent pass will detect future imports that don't
-	   and skip doc strings. */
+	   appear at the beginning of the file.  There's one case,
-	case expr_stmt:
+	   however, that is easier to handl here: A series of imports
-	case testlist:
+	   joined by semi-colons, where the first import is a future
-	case test:
+	   statement but some subsequent import has the future form
-	case and_test:
+	   but is preceded by a regular import.
-	case not_test:
+	*/
-	case comparison:
+	   
-	case expr:
+
-	case xor_expr:
+	for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
-	case and_expr:
+		stmt_ty s = asdl_seq_GET(mod->v.Module.body, i);
-	case shift_expr:
+
-	case arith_expr:
+		if (done && s->lineno > prev_line)
-	case term:
+			return 1;
-	case factor:
+		prev_line = s->lineno;
-	case power:
+
-		if (NCH(n) == 1) {
+		/* The tests below will return from this function unless it is
-			n = CHILD(n, 0);
+		   still possible to find a future statement.  The only things
-			goto loop;
+		   that can precede a future statement are another future
+		   statement and a doc string.
+		*/
+
+		if (s->kind == ImportFrom_kind) {
+			if (s->v.ImportFrom.module == future) {
+				if (done) {
+					PyErr_SetString(PyExc_SyntaxError,
+							ERR_LATE_FUTURE);
+					PyErr_SyntaxLocation(filename, 
+							     s->lineno);
+					return 0;
+				}
+				if (!future_check_features(ff, s, filename))
+					return 0;
+				ff->ff_lineno = s->lineno;
+			}
+			else
+				done = 1;
 		}
-                ff->ff_last_lineno = n->n_lineno;
+		else if (s->kind == Expr_kind && !found_docstring) {
-		break;
+			expr_ty e = s->v.Expr.value;
-
+			if (e->kind != Str_kind)
-	case atom:
+				done = 1;
-		if (TYPE(CHILD(n, 0)) == STRING 
+			else
-		    && ff->ff_found_docstring == 0) {
+				found_docstring = 1;
-			ff->ff_found_docstring = 1;
-			return 0;
 		}
-		ff->ff_last_lineno = n->n_lineno;
+		else
-		return 0;
+			done = 1;
-
-	default:
-		ff->ff_last_lineno = n->n_lineno;
-		return 0;
 	}
-	return 0;
+	return 1;
 }
 
+
 PyFutureFeatures *
-PyNode_Future(node *n, const char *filename)
+PyFuture_FromAST(mod_ty mod, const char *filename)
 {
 	PyFutureFeatures *ff;
 
 	ff = (PyFutureFeatures *)PyMem_Malloc(sizeof(PyFutureFeatures));
 	if (ff == NULL)
 		return NULL;
-	ff->ff_found_docstring = 0;
-	ff->ff_last_lineno = -1;
 	ff->ff_features = 0;
+	ff->ff_lineno = -1;
 
-	if (future_parse(ff, n, filename) < 0) {
+	if (!future_parse(ff, mod, filename)) {
 		PyMem_Free((void *)ff);
 		return NULL;
 	}

Python/import.c

@@ -3,10 +3,11 @@
 
 #include "Python.h"
 
-#include "node.h"
+#include "Python-ast.h"
-#include "token.h"
+#include "pythonrun.h"
 #include "errcode.h"
 #include "marshal.h"
+#include "code.h"
 #include "compile.h"
 #include "eval.h"
 #include "osdefs.h"
@@ -766,17 +767,17 @@ load_compiled_module(char *name, char *cpathname, FILE *fp)
 /* Parse a source file and return the corresponding code object */
 
 static PyCodeObject *
-parse_source_module(char *pathname, FILE *fp)
+parse_source_module(const char *pathname, FILE *fp)
 {
-	PyCodeObject *co;
+	PyCodeObject *co = NULL;
-	node *n;
+	mod_ty mod;
-
-	n = PyParser_SimpleParseFile(fp, pathname, Py_file_input);
-	if (n == NULL)
-		return NULL;
-	co = PyNode_Compile(n, pathname);
-	PyNode_Free(n);
 
+	mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, 
+				   NULL);
+	if (mod) {
+		co = PyAST_Compile(mod, pathname, NULL);
+		free_mod(mod);
+	}
 	return co;
 }
 

Python/marshal.c

@@ -6,6 +6,7 @@
 
 #include "Python.h"
 #include "longintrepr.h"
+#include "code.h"
 #include "compile.h"
 #include "marshal.h"
 

Python/pythonrun.c

@@ -3,13 +3,16 @@
 
 #include "Python.h"
 
+#include "Python-ast.h"
 #include "grammar.h"
 #include "node.h"
 #include "token.h"
 #include "parsetok.h"
 #include "errcode.h"
+#include "code.h"
 #include "compile.h"
 #include "symtable.h"
+#include "ast.h"
 #include "eval.h"
 #include "marshal.h"
 
@@ -32,9 +35,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */
 /* Forward */
 static void initmain(void);
 static void initsite(void);
-static PyObject *run_err_node(node *, const char *, PyObject *, PyObject *,
+static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *,
 			      PyCompilerFlags *);
-static PyObject *run_node(node *, const char *, PyObject *, PyObject *,
+static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
 			  PyCompilerFlags *);
 static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
 			      PyCompilerFlags *);
@@ -634,25 +637,7 @@ initsite(void)
 /* Parse input from a file and execute it */
 
 int
-PyRun_AnyFile(FILE *fp, const char *filename)
+PyRun_AnyFileExFlags(FILE *fp, char *filename, int closeit, 
-{
-	return PyRun_AnyFileExFlags(fp, filename, 0, NULL);
-}
-
-int
-PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
-{
-	return PyRun_AnyFileExFlags(fp, filename, 0, flags);
-}
-
-int
-PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit)
-{
-	return PyRun_AnyFileExFlags(fp, filename, closeit, NULL);
-}
-
-int
-PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
 		     PyCompilerFlags *flags)
 {
 	if (filename == NULL)
@@ -667,12 +652,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
 		return PyRun_SimpleFileExFlags(fp, filename, closeit, flags);
 }
 
-int
-PyRun_InteractiveLoop(FILE *fp, const char *filename)
-{
-	return PyRun_InteractiveLoopFlags(fp, filename, NULL);
-}
-
 int
 PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
@@ -708,12 +687,6 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 	}
 }
 
-int
-PyRun_InteractiveOne(FILE *fp, const char *filename)
-{
-	return PyRun_InteractiveOneFlags(fp, filename, NULL);
-}
-
 /* compute parser flags based on compiler flags */
 #define PARSER_FLAGS(flags) \
 	(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
@@ -723,9 +696,9 @@ int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
 	PyObject *m, *d, *v, *w;
-	node *n;
+	mod_ty mod;
-	perrdetail err;
 	char *ps1 = "", *ps2 = "";
+	int errcode = 0;
 
 	v = PySys_GetObject("ps1");
 	if (v != NULL) {
@@ -743,26 +716,25 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 		else if (PyString_Check(w))
 			ps2 = PyString_AsString(w);
 	}
-	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+	mod = PyParser_ASTFromFile(fp, filename, 
-			    	    Py_single_input, ps1, ps2, &err,
+				   Py_single_input, ps1, ps2,
-			    	    PARSER_FLAGS(flags));
+				   flags, &errcode);
 	Py_XDECREF(v);
 	Py_XDECREF(w);
-	if (n == NULL) {
+	if (mod == NULL) {
-		if (err.error == E_EOF) {
+		if (errcode == E_EOF) {
-			if (err.text)
+			PyErr_Clear();
-				PyMem_DEL(err.text);
 			return E_EOF;
 		}
-		err_input(&err);
 		PyErr_Print();
-		return err.error;
+		return -1;
 	}
 	m = PyImport_AddModule("__main__");
 	if (m == NULL)
 		return -1;
 	d = PyModule_GetDict(m);
-	v = run_node(n, filename, d, d, flags);
+	v = run_mod(mod, filename, d, d, flags);
+	free_mod(mod);
 	if (v == NULL) {
 		PyErr_Print();
 		return -1;
@@ -773,12 +745,6 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 	return 0;
 }
 
-int
-PyRun_SimpleFile(FILE *fp, const char *filename)
-{
-	return PyRun_SimpleFileEx(fp, filename, 0);
-}
-
 /* Check whether a file maybe a pyc file: Look at the extension,
    the file type, and, if we may close it, at the first few bytes. */
 
@@ -819,12 +785,6 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
 	return 0;
 }
 
-int
-PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit)
-{
-	return PyRun_SimpleFileExFlags(fp, filename, closeit, NULL);
-}
-
 int
 PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 			PyCompilerFlags *flags)
@@ -873,12 +833,6 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 	return 0;
 }
 
-int
-PyRun_SimpleString(const char *command)
-{
-	return PyRun_SimpleStringFlags(command, NULL);
-}
-
 int
 PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
 {
@@ -1054,6 +1008,8 @@ PyErr_PrintEx(int set_sys_last_vars)
 		handle_system_exit();
 	}
 	PyErr_Fetch(&exception, &v, &tb);
+	if (exception == NULL)
+		return;
 	PyErr_NormalizeException(&exception, &v, &tb);
 	if (exception == NULL)
 		return;
@@ -1195,74 +1151,48 @@ void PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb)
 }
 
 PyObject *
-PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals)
+PyRun_StringFlags(const char *str, int start, PyObject *globals, 
+		  PyObject *locals, PyCompilerFlags *flags)
 {
-	return run_err_node(PyParser_SimpleParseString(str, start),
+	PyObject *ret;
-			    "<string>", globals, locals, NULL);
+	mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags);
-}
+	ret = run_err_mod(mod, "<string>", globals, locals, flags);
-
+	free_mod(mod);
-PyObject *
+	return ret;
-PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals,
-	   PyObject *locals)
-{
-	return PyRun_FileEx(fp, filename, start, globals, locals, 0);
-}
-
-PyObject *
-PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals,
-	     PyObject *locals, int closeit)
-{
-	node *n = PyParser_SimpleParseFile(fp, filename, start);
-	if (closeit)
-		fclose(fp);
-	return run_err_node(n, filename, globals, locals, NULL);
-}
-
-PyObject *
-PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals,
-		  PyCompilerFlags *flags)
-{
-	return run_err_node(PyParser_SimpleParseStringFlags(
-				    str, start, PARSER_FLAGS(flags)),
-			    "<string>", globals, locals, flags);
-}
-
-PyObject *
-PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals,
-		PyObject *locals, PyCompilerFlags *flags)
-{
-	return PyRun_FileExFlags(fp, filename, start, globals, locals, 0,
-				 flags);
 }
 
 PyObject *
 PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
 		  PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
-	node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
+	PyObject *ret;
-						PARSER_FLAGS(flags));
+	mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
+					  flags, NULL);
+	if (mod == NULL)
+		return NULL;
 	if (closeit)
 		fclose(fp);
-	return run_err_node(n, filename, globals, locals, flags);
+	ret = run_err_mod(mod, filename, globals, locals, flags);
+	free_mod(mod);
+	return ret;
 }
 
 static PyObject *
-run_err_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
+run_err_mod(mod_ty mod, const char *filename, PyObject *globals, 
-	     PyCompilerFlags *flags)
+	    PyObject *locals, PyCompilerFlags *flags)
 {
-	if (n == NULL)
+	if (mod == NULL)
 		return  NULL;
-	return run_node(n, filename, globals, locals, flags);
+	return run_mod(mod, filename, globals, locals, flags);
 }
 
 static PyObject *
-run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
+run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
 	 PyCompilerFlags *flags)
 {
 	PyCodeObject *co;
 	PyObject *v;
-	co = PyNode_CompileFlags(n, filename, flags);
+	co = PyAST_Compile(mod, filename, flags);
-	PyNode_Free(n);
 	if (co == NULL)
 		return NULL;
 	v = PyEval_EvalCode(co, globals, locals);
@@ -1271,8 +1201,8 @@ run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
 }
 
 static PyObject *
-run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals,
+run_pyc_file(FILE *fp, const char *filename, PyObject *globals, 
-	     PyCompilerFlags *flags)
+	     PyObject *locals, PyCompilerFlags *flags)
 {
 	PyCodeObject *co;
 	PyObject *v;
@@ -1302,42 +1232,78 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals
 	return v;
 }
 
-PyObject *
-Py_CompileString(const char *str, const char *filename, int start)
-{
-	return Py_CompileStringFlags(str, filename, start, NULL);
-}
-
 PyObject *
 Py_CompileStringFlags(const char *str, const char *filename, int start,
 		      PyCompilerFlags *flags)
 {
-	node *n;
+	mod_ty mod;
 	PyCodeObject *co;
-
+	mod = PyParser_ASTFromString(str, filename, start, flags);
-	n = PyParser_SimpleParseStringFlagsFilename(str, filename, start,
+	if (mod == NULL)
-						    PARSER_FLAGS(flags));
-	if (n == NULL)
 		return NULL;
-	co = PyNode_CompileFlags(n, filename, flags);
+	co = PyAST_Compile(mod, filename, flags);
-	PyNode_Free(n);
+	free_mod(mod);
 	return (PyObject *)co;
 }
 
 struct symtable *
 Py_SymtableString(const char *str, const char *filename, int start)
 {
-	node *n;
+	mod_ty mod;
 	struct symtable *st;
-	n = PyParser_SimpleParseStringFlagsFilename(str, filename,
+
-						    start, 0);
+	mod = PyParser_ASTFromString(str, filename, start, NULL);
-	if (n == NULL)
+	if (mod == NULL)
 		return NULL;
-	st = PyNode_CompileSymtable(n, filename);
+	st = PySymtable_Build(mod, filename, 0);
-	PyNode_Free(n);
+	free_mod(mod);
 	return st;
 }
 
+/* Preferred access to parser is through AST. */
+mod_ty
+PyParser_ASTFromString(const char *s, const char *filename, int start, 
+		       PyCompilerFlags *flags)
+{
+	node *n;
+	mod_ty mod;
+	perrdetail err;
+	n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar,
+					      start, &err, 
+					      PARSER_FLAGS(flags));
+	if (n) {
+		mod = PyAST_FromNode(n, flags, filename);
+		PyNode_Free(n);
+		return mod;
+	}
+	else {
+		err_input(&err);
+		return NULL;
+	}
+}
+
+mod_ty
+PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, 
+		     char *ps2, PyCompilerFlags *flags, int *errcode)
+{
+	node *n;
+	mod_ty mod;
+	perrdetail err;
+	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start, 
+				    ps1, ps2, &err, PARSER_FLAGS(flags));
+	if (n) {
+		mod = PyAST_FromNode(n, flags, filename);
+		PyNode_Free(n);
+		return mod;
+	}
+	else {
+		err_input(&err);
+		if (errcode)
+			*errcode = err.error;
+		return NULL;
+	}
+}
+
 /* Simplified interface to parsefile -- return node or set exception */
 
 node *
@@ -1349,15 +1315,10 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla
 					(char *)0, (char *)0, &err, flags);
 	if (n == NULL)
 		err_input(&err);
+		
 	return n;
 }
 
-node *
-PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
-{
-	return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
-}
-
 /* Simplified interface to parsestring -- return node or set exception */
 
 node *
@@ -1372,12 +1333,6 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags)
 	return n;
 }
 
-node *
-PyParser_SimpleParseString(const char *str, int start)
-{
-	return PyParser_SimpleParseStringFlags(str, start, 0);
-}
-
 node *
 PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
 					int start, int flags)
@@ -1418,12 +1373,6 @@ err_input(perrdetail *err)
 	PyObject* u = NULL;
 	char *msg = NULL;
 	errtype = PyExc_SyntaxError;
-	v = Py_BuildValue("(ziiz)", err->filename,
-			    err->lineno, err->offset, err->text);
-	if (err->text != NULL) {
-		PyMem_DEL(err->text);
-		err->text = NULL;
-	}
 	switch (err->error) {
 	case E_SYNTAX:
 		errtype = PyExc_IndentationError;
@@ -1450,11 +1399,9 @@ err_input(perrdetail *err)
 	case E_INTR:
 		if (!PyErr_Occurred())
 			PyErr_SetNone(PyExc_KeyboardInterrupt);
-		Py_XDECREF(v);
 		return;
 	case E_NOMEM:
 		PyErr_NoMemory();
-		Py_XDECREF(v);
 		return;
 	case E_EOF:
 		msg = "unexpected EOF while parsing";
@@ -1498,7 +1445,15 @@ err_input(perrdetail *err)
 		msg = "unknown parsing error";
 		break;
 	}
-	w = Py_BuildValue("(sO)", msg, v);
+	v = Py_BuildValue("(ziiz)", err->filename,
+			  err->lineno, err->offset, err->text);
+ 	if (err->text != NULL) {
+		PyMem_DEL(err->text);
+		err->text = NULL;
+	}
+	w = NULL;
+	if (v != NULL)
+		w = Py_BuildValue("(sO)", msg, v);
 	Py_XDECREF(u);
 	Py_XDECREF(v);
 	PyErr_SetObject(errtype, w);
@@ -1687,3 +1642,20 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler)
 	return oldhandler;
 #endif
 }
+
+/* Deprecated C API functions still provided for binary compatiblity */
+
+#undef PyParser_SimpleParseFile
+#undef PyParser_SimpleParseString
+
+node *
+PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
+{
+	return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
+}
+
+node *
+PyParser_SimpleParseString(const char *str, int start)
+{
+	return PyParser_SimpleParseStringFlags(str, start, 0);
+}

Python/sysmodule.c

@@ -15,7 +15,7 @@ Data members:
 */
 
 #include "Python.h"
-#include "compile.h"
+#include "code.h"
 #include "frameobject.h"
 #include "eval.h"
 

Python/traceback.c

@@ -3,7 +3,7 @@
 
 #include "Python.h"
 
-#include "compile.h"
+#include "code.h"
 #include "frameobject.h"
 #include "structmember.h"
 #include "osdefs.h"

Tools/compiler/dumppyc.py

@@ -28,7 +28,7 @@ def walk(co, match=None):
         if type(obj) == types.CodeType:
             walk(obj, match)
 
-def main(filename, codename=None):
+def load(filename, codename=None):
     co = loadCode(filename)
     walk(co, codename)
 
@@ -39,6 +39,9 @@ if __name__ == "__main__":
     else:
         filename = sys.argv[1]
         codename = None
-    if filename.endswith('.py') and os.path.exists(filename+"c"):
+    if filename.endswith('.py'):
-        filename += "c"
+        buf = open(filename).read()
-    main(filename, codename)
+        co = compile(buf, filename, "exec")
+        walk(co)
+    else:   
+        load(filename, codename)