Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Merge ast-branch to head 

This change implements a new bytecode compiler, based on a
transformation of the parse tree to an abstract syntax defined in
Parser/Python.asdl.

The compiler implementation is not complete, but it is in stable
enough shape to run the entire test suite excepting two disabled
tests.
This commit is contained in:
Jeremy Hylton 2005-10-20 19:59:25 +00:00
parent 2cb94aba12
commit 3e0055f8c6
54 changed files with 13675 additions and 6810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

418
Include/Python-ast.h Normal file
View File

@ -0,0 +1,418 @@
/* File automatically generated by ../Parser/asdl_c.py */
#include "asdl.h"
typedef struct _mod *mod_ty;
typedef struct _stmt *stmt_ty;
typedef struct _expr *expr_ty;
typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5,
Param=6 } expr_context_ty;
typedef struct _slice *slice_ty;
typedef enum _boolop { And=1, Or=2 } boolop_ty;
typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7,
RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 }
operator_ty;
typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty;
typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8,
In=9, NotIn=10 } cmpop_ty;
typedef struct _comprehension *comprehension_ty;
typedef struct _excepthandler *excepthandler_ty;
typedef struct _arguments *arguments_ty;
typedef struct _keyword *keyword_ty;
typedef struct _alias *alias_ty;
struct _mod {
enum { Module_kind=1, Interactive_kind=2, Expression_kind=3,
Suite_kind=4 } kind;
union {
struct {
asdl_seq *body;
} Module;
struct {
asdl_seq *body;
} Interactive;
struct {
expr_ty body;
} Expression;
struct {
asdl_seq *body;
} Suite;
} v;
};
struct _stmt {
enum { FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
Delete_kind=4, Assign_kind=5, AugAssign_kind=6, Print_kind=7,
For_kind=8, While_kind=9, If_kind=10, Raise_kind=11,
TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
Import_kind=15, ImportFrom_kind=16, Exec_kind=17,
Global_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
Continue_kind=22 } kind;
union {
struct {
identifier name;
arguments_ty args;
asdl_seq *body;
asdl_seq *decorators;
} FunctionDef;
struct {
identifier name;
asdl_seq *bases;
asdl_seq *body;
} ClassDef;
struct {
expr_ty value;
} Return;
struct {
asdl_seq *targets;
} Delete;
struct {
asdl_seq *targets;
expr_ty value;
} Assign;
struct {
expr_ty target;
operator_ty op;
expr_ty value;
} AugAssign;
struct {
expr_ty dest;
asdl_seq *values;
bool nl;
} Print;
struct {
expr_ty target;
expr_ty iter;
asdl_seq *body;
asdl_seq *orelse;
} For;
struct {
expr_ty test;
asdl_seq *body;
asdl_seq *orelse;
} While;
struct {
expr_ty test;
asdl_seq *body;
asdl_seq *orelse;
} If;
struct {
expr_ty type;
expr_ty inst;
expr_ty tback;
} Raise;
struct {
asdl_seq *body;
asdl_seq *handlers;
asdl_seq *orelse;
} TryExcept;
struct {
asdl_seq *body;
asdl_seq *finalbody;
} TryFinally;
struct {
expr_ty test;
expr_ty msg;
} Assert;
struct {
asdl_seq *names;
} Import;
struct {
identifier module;
asdl_seq *names;
} ImportFrom;
struct {
expr_ty body;
expr_ty globals;
expr_ty locals;
} Exec;
struct {
asdl_seq *names;
} Global;
struct {
expr_ty value;
} Expr;
} v;
int lineno;
};
struct _expr {
enum { BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
Dict_kind=5, ListComp_kind=6, GeneratorExp_kind=7, Yield_kind=8,
Compare_kind=9, Call_kind=10, Repr_kind=11, Num_kind=12,
Str_kind=13, Attribute_kind=14, Subscript_kind=15, Name_kind=16,
List_kind=17, Tuple_kind=18 } kind;
union {
struct {
boolop_ty op;
asdl_seq *values;
} BoolOp;
struct {
expr_ty left;
operator_ty op;
expr_ty right;
} BinOp;
struct {
unaryop_ty op;
expr_ty operand;
} UnaryOp;
struct {
arguments_ty args;
expr_ty body;
} Lambda;
struct {
asdl_seq *keys;
asdl_seq *values;
} Dict;
struct {
expr_ty elt;
asdl_seq *generators;
} ListComp;
struct {
expr_ty elt;
asdl_seq *generators;
} GeneratorExp;
struct {
expr_ty value;
} Yield;
struct {
expr_ty left;
asdl_seq *ops;
asdl_seq *comparators;
} Compare;
struct {
expr_ty func;
asdl_seq *args;
asdl_seq *keywords;
expr_ty starargs;
expr_ty kwargs;
} Call;
struct {
expr_ty value;
} Repr;
struct {
object n;
} Num;
struct {
string s;
} Str;
struct {
expr_ty value;
identifier attr;
expr_context_ty ctx;
} Attribute;
struct {
expr_ty value;
slice_ty slice;
expr_context_ty ctx;
} Subscript;
struct {
identifier id;
expr_context_ty ctx;
} Name;
struct {
asdl_seq *elts;
expr_context_ty ctx;
} List;
struct {
asdl_seq *elts;
expr_context_ty ctx;
} Tuple;
} v;
int lineno;
};
struct _slice {
enum { Ellipsis_kind=1, Slice_kind=2, ExtSlice_kind=3, Index_kind=4 }
kind;
union {
struct {
expr_ty lower;
expr_ty upper;
expr_ty step;
} Slice;
struct {
asdl_seq *dims;
} ExtSlice;
struct {
expr_ty value;
} Index;
} v;
};
struct _comprehension {
expr_ty target;
expr_ty iter;
asdl_seq *ifs;
};
struct _excepthandler {
expr_ty type;
expr_ty name;
asdl_seq *body;
};
struct _arguments {
asdl_seq *args;
identifier vararg;
identifier kwarg;
asdl_seq *defaults;
};
struct _keyword {
identifier arg;
expr_ty value;
};
struct _alias {
identifier name;
identifier asname;
};
mod_ty Module(asdl_seq * body);
mod_ty Interactive(asdl_seq * body);
mod_ty Expression(expr_ty body);
mod_ty Suite(asdl_seq * body);
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorators, int lineno);
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
lineno);
stmt_ty Return(expr_ty value, int lineno);
stmt_ty Delete(asdl_seq * targets, int lineno);
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno);
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno);
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno);
stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
int lineno);
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno);
stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
lineno);
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno);
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno);
stmt_ty Import(asdl_seq * names, int lineno);
stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno);
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno);
stmt_ty Global(asdl_seq * names, int lineno);
stmt_ty Expr(expr_ty value, int lineno);
stmt_ty Pass(int lineno);
stmt_ty Break(int lineno);
stmt_ty Continue(int lineno);
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno);
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno);
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno);
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno);
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno);
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno);
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno);
expr_ty Yield(expr_ty value, int lineno);
expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
lineno);
expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
starargs, expr_ty kwargs, int lineno);
expr_ty Repr(expr_ty value, int lineno);
expr_ty Num(object n, int lineno);
expr_ty Str(string s, int lineno);
expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
lineno);
expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
lineno);
expr_ty Name(identifier id, expr_context_ty ctx, int lineno);
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno);
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno);
slice_ty Ellipsis(void);
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step);
slice_ty ExtSlice(asdl_seq * dims);
slice_ty Index(expr_ty value);
comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs);
excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body);
arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
asdl_seq * defaults);
keyword_ty keyword(identifier arg, expr_ty value);
alias_ty alias(identifier name, identifier asname);
void free_mod(mod_ty);
void free_stmt(stmt_ty);
void free_expr(expr_ty);
void free_expr_context(expr_context_ty);
void free_slice(slice_ty);
void free_boolop(boolop_ty);
void free_operator(operator_ty);
void free_unaryop(unaryop_ty);
void free_cmpop(cmpop_ty);
void free_comprehension(comprehension_ty);
void free_excepthandler(excepthandler_ty);
void free_arguments(arguments_ty);
void free_keyword(keyword_ty);
void free_alias(alias_ty);
int marshal_write_mod(PyObject **, int *, mod_ty);
int marshal_write_stmt(PyObject **, int *, stmt_ty);
int marshal_write_expr(PyObject **, int *, expr_ty);
int marshal_write_expr_context(PyObject **, int *, expr_context_ty);
int marshal_write_slice(PyObject **, int *, slice_ty);
int marshal_write_boolop(PyObject **, int *, boolop_ty);
int marshal_write_operator(PyObject **, int *, operator_ty);
int marshal_write_unaryop(PyObject **, int *, unaryop_ty);
int marshal_write_cmpop(PyObject **, int *, cmpop_ty);
int marshal_write_comprehension(PyObject **, int *, comprehension_ty);
int marshal_write_excepthandler(PyObject **, int *, excepthandler_ty);
int marshal_write_arguments(PyObject **, int *, arguments_ty);
int marshal_write_keyword(PyObject **, int *, keyword_ty);
int marshal_write_alias(PyObject **, int *, alias_ty);

3
Include/Python.h
View File

@ -128,8 +128,7 @@
#include "pystrtod.h"
/* _Py_Mangle is defined in compile.c */
PyAPI_FUNC(int) _Py_Mangle(char *p, char *name, \
char *buffer, size_t maxlen);
PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
/* PyArg_GetInt is deprecated and should not be used, use PyArg_Parse(). */
#define PyArg_GetInt(v, a) PyArg_Parse((v), "i", (a))

54
Include/asdl.h Normal file
View File

@ -0,0 +1,54 @@
#ifndef Py_ASDL_H
#define Py_ASDL_H
typedef PyObject * identifier;
typedef PyObject * string;
typedef PyObject * object;
typedef enum {false, true} bool;
/* It would be nice if the code generated by asdl_c.py was completely
independent of Python, but it is a goal the requires too much work
at this stage. So, for example, I'll represent identifiers as
interned Python strings.
*/
/* XXX A sequence should be typed so that its use can be typechecked. */
/* XXX We shouldn't pay for offset when we don't need APPEND. */
typedef struct {
int size;
int offset;
void *elements[1];
} asdl_seq;
asdl_seq *asdl_seq_new(int size);
void asdl_seq_free(asdl_seq *);
#ifdef Py_DEBUG
#define asdl_seq_GET(S, I) (S)->elements[(I)]
#define asdl_seq_SET(S, I, V) { \
int _asdl_i = (I); \
assert((S) && _asdl_i < (S)->size); \
(S)->elements[_asdl_i] = (V); \
}
#define asdl_seq_APPEND(S, V) { \
assert((S) && (S)->offset < (S)->size); \
(S)->elements[(S)->offset++] = (V); \
}
#else
#define asdl_seq_GET(S, I) (S)->elements[(I)]
#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
#define asdl_seq_APPEND(S, V) (S)->elements[(S)->offset++] = (V)
#endif
#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
/* Routines to marshal the basic types. */
int marshal_write_int(PyObject **, int *, int);
int marshal_write_bool(PyObject **, int *, bool);
int marshal_write_identifier(PyObject **, int *, identifier);
int marshal_write_string(PyObject **, int *, string);
int marshal_write_object(PyObject **, int *, object);
#endif /* !Py_ASDL_H */

13
Include/ast.h Normal file
View File

@ -0,0 +1,13 @@
#ifndef Py_AST_H
#define Py_AST_H
#ifdef __cplusplus
extern "C" {
#endif
extern DL_IMPORT(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
const char *);
#ifdef __cplusplus
}
#endif
#endif /* !Py_AST_H */

73
Include/code.h Normal file
View File

@ -0,0 +1,73 @@
/* Definitions for bytecode */
#ifndef Py_CODE_H
#define Py_CODE_H
#ifdef __cplusplus
extern "C" {
#endif
/* Bytecode object */
typedef struct {
PyObject_HEAD
int co_argcount; /* #arguments, except *args */
int co_nlocals; /* #local variables */
int co_stacksize; /* #entries needed for evaluation stack */
int co_flags; /* CO_..., see below */
PyObject *co_code; /* instruction opcodes */
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
/* The rest doesn't count for hash/cmp */
PyObject *co_filename; /* string (where it was loaded from) */
PyObject *co_name; /* string (name, for reference) */
int co_firstlineno; /* first source line number */
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
} PyCodeObject;
/* Masks for co_flags above */
#define CO_OPTIMIZED 0x0001
#define CO_NEWLOCALS 0x0002
#define CO_VARARGS 0x0004
#define CO_VARKEYWORDS 0x0008
#define CO_NESTED 0x0010
#define CO_GENERATOR 0x0020
/* The CO_NOFREE flag is set if there are no free or cell variables.
This information is redundant, but it allows a single flag test
to determine whether there is any extra work to be done when the
call frame it setup.
*/
#define CO_NOFREE 0x0040
/* XXX Temporary hack. Until generators are a permanent part of the
language, we need a way for a code object to record that generators
were *possible* when it was compiled. This is so code dynamically
compiled *by* a code object knows whether to allow yield stmts. In
effect, this passes on the "from __future__ import generators" state
in effect when the code block was compiled. */
#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */
#define CO_FUTURE_DIVISION 0x2000
#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
extern DL_IMPORT(PyTypeObject) PyCode_Type;
#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
/* Public interface */
DL_IMPORT(PyCodeObject *) PyCode_New(
int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *);
/* same as struct above */
DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
/* for internal use only */
#define _PyCode_GETCODEPTR(co, pp) \
((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \
((co)->co_code, 0, (void **)(pp)))
#ifdef __cplusplus
}
#endif
#endif /* !Py_CODE_H */

71
Include/compile.h
View File

@ -1,5 +1,6 @@
/* Definitions for bytecode */
#ifndef Py_CODE_H
#include "code.h"
#endif
#ifndef Py_COMPILE_H
#define Py_COMPILE_H
@ -7,55 +8,6 @@
extern "C" {
#endif
/* Bytecode object */
typedef struct {
PyObject_HEAD
int co_argcount; /* #arguments, except *args */
int co_nlocals; /* #local variables */
int co_stacksize; /* #entries needed for evaluation stack */
int co_flags; /* CO_..., see below */
PyObject *co_code; /* instruction opcodes */
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
/* The rest doesn't count for hash/cmp */
PyObject *co_filename; /* string (where it was loaded from) */
PyObject *co_name; /* string (name, for reference) */
int co_firstlineno; /* first source line number */
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
} PyCodeObject;
/* Masks for co_flags above */
#define CO_OPTIMIZED 0x0001
#define CO_NEWLOCALS 0x0002
#define CO_VARARGS 0x0004
#define CO_VARKEYWORDS 0x0008
#define CO_NESTED 0x0010
#define CO_GENERATOR 0x0020
/* The CO_NOFREE flag is set if there are no free or cell variables.
This information is redundant, but it allows a single flag test
to determine whether there is any extra work to be done when the
call frame it setup.
*/
#define CO_NOFREE 0x0040
/* XXX Temporary hack. Until generators are a permanent part of the
language, we need a way for a code object to record that generators
were *possible* when it was compiled. This is so code dynamically
compiled *by* a code object knows whether to allow yield stmts. In
effect, this passes on the "from __future__ import generators" state
in effect when the code block was compiled. */
#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */
#define CO_FUTURE_DIVISION 0x2000
PyAPI_DATA(PyTypeObject) PyCode_Type;
#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
/* Public interface */
struct _node; /* Declare the existence of this type */
PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
@ -68,19 +20,22 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
/* Future feature support */
typedef struct {
int ff_found_docstring;
int ff_last_lineno;
int ff_features;
int ff_features; /* flags set by future statements */
int ff_lineno; /* line number of last future statement */
} PyFutureFeatures;
PyAPI_FUNC(PyFutureFeatures *) PyNode_Future(struct _node *, const char *);
PyAPI_FUNC(PyCodeObject *) PyNode_CompileFlags(struct _node *, const char *,
PyCompilerFlags *);
#define FUTURE_NESTED_SCOPES "nested_scopes"
#define FUTURE_GENERATORS "generators"
#define FUTURE_DIVISION "division"
struct _mod; /* Declare the existence of this type */
DL_IMPORT(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
PyCompilerFlags *);
DL_IMPORT(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
#define ERR_LATE_FUTURE \
"from __future__ imports must occur at the beginning of the file"
#ifdef __cplusplus
}
#endif

1
Include/pyport.h
View File

@ -583,6 +583,7 @@ typedef struct fd_set {
#ifndef INT_MAX
#define INT_MAX 2147483647
#define INT_MIN (-INT_MAX - 1)
#endif
#ifndef LONG_MAX

70
Include/pythonrun.h
View File

@ -29,46 +29,37 @@ PyAPI_FUNC(int) Py_IsInitialized(void);
PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
PyAPI_FUNC(int) PyRun_AnyFile(FILE *, const char *);
PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *, const char *, int);
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_SimpleString(const char *);
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, char *, int, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_SimpleFile(FILE *, const char *);
PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *, const char *, int);
PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *, const char *);
PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *, const char *);
PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
PyAPI_FUNC(struct _node *) PyParser_SimpleParseString(const char *, int);
PyAPI_FUNC(struct _node *) PyParser_SimpleParseFile(FILE *, const char *, int);
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int);
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *,
const char *,
int,
int);
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
int, PyCompilerFlags *flags);
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
char *, char *,
PyCompilerFlags *, int *);
#define PyParser_SimpleParseString(S, B) \
PyParser_SimpleParseStringFlags(S, B, 0)
#define PyParser_SimpleParseFile(FP, S, B) \
PyParser_SimpleParseFileFlags(FP, S, B, 0)
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int,
int);
PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
int, int);
PyAPI_FUNC(PyObject *) PyRun_String(const char *, int, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyRun_File(FILE *, const char *, int, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *, const char *, int,
PyObject *, PyObject *, int);
PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *,
PyCompilerFlags *);
PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *, const char *, int, PyObject *,
PyObject *, PyCompilerFlags *);
PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, PyObject *,
PyObject *, int, PyCompilerFlags *);
PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
PyObject *, PyCompilerFlags *);
PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
PyObject *, PyObject *, int,
PyCompilerFlags *);
#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL)
PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int,
PyCompilerFlags *);
PyCompilerFlags *);
PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
PyAPI_FUNC(void) PyErr_Print(void);
@ -84,6 +75,25 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
/* Bootstrap */
PyAPI_FUNC(int) Py_Main(int argc, char **argv);
/* Use macros for a bunch of old variants */
#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL)
#define PyRun_AnyFileEx(fp, name, closeit) \
PyRun_AnyFileExFlags(fp, name, closeit, NULL)
#define PyRun_AnyFileFlags(fp, name, flags) \
PyRun_AnyFileExFlags(fp, name, 0, flags)
#define PyRun_SimpleString(s, f) PyRunSimpleStringFlags(s, f, NULL)
#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL)
#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL)
#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL)
#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL)
#define PyRun_File(fp, p, s, g, l) \
PyRun_FileExFlags(fp, p, s, g, l, 0, NULL)
#define PyRun_FileEx(fp, p, s, g, l, c) \
PyRun_FileExFlags(fp, p, s, g, l, c, NULL)
#define PyRun_FileFlags(fp, p, s, g, l, flags) \
PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
/* In getpath.c */
PyAPI_FUNC(char *) Py_GetProgramFullPath(void);
PyAPI_FUNC(char *) Py_GetPrefix(void);

77
Include/symtable.h
View File

@ -4,65 +4,60 @@
extern "C" {
#endif
/* A symbol table is constructed each time PyNode_Compile() is
called. The table walks the entire parse tree and identifies each
use or definition of a variable.
The symbol table contains a dictionary for each code block in a
module: The symbol dictionary for the block. They keys of these
dictionaries are the name of all variables used or defined in the
block; the integer values are used to store several flags,
e.g. DEF_PARAM indicates that a variable is a parameter to a
function.
*/
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
block_ty;
struct _symtable_entry;
struct symtable {
int st_pass; /* pass == 1 or 2 */
const char *st_filename; /* name of file being compiled */
struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* module entry */
PyObject *st_symbols; /* dictionary of symbol table entries */
PyObject *st_stack; /* stack of namespace info */
PyObject *st_global; /* borrowed ref to MODULE in st_symbols */
int st_nscopes; /* number of scopes */
int st_errors; /* number of errors */
int st_nblocks; /* number of blocks */
char *st_private; /* name of current class or NULL */
int st_tmpname; /* temporary name counter */
PyFutureFeatures *st_future; /* module's future features */
};
typedef struct _symtable_entry {
PyObject_HEAD
PyObject *ste_id; /* int: key in st_symbols) */
PyObject *ste_symbols; /* dict: name to flags) */
PyObject *ste_name; /* string: name of scope */
PyObject *ste_id; /* int: key in st_symbols */
PyObject *ste_symbols; /* dict: name to flags */
PyObject *ste_name; /* string: name of block */
PyObject *ste_varnames; /* list of variable names */
PyObject *ste_children; /* list of child ids */
int ste_type; /* module, class, or function */
int ste_lineno; /* first line of scope */
int ste_optimized; /* true if namespace can't be optimized */
int ste_nested; /* true if scope is nested */
int ste_child_free; /* true if a child scope has free variables,
block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
int ste_nested : 1; /* true if block is nested */
int ste_free : 1; /* true if block has free variables */
int ste_child_free : 1; /* true if a child block has free variables,
including free refs to globals */
int ste_generator; /* true if namespace is a generator */
int ste_generator : 1; /* true if namespace is a generator */
int ste_varargs : 1; /* true if block has varargs */
int ste_varkeywords : 1; /* true if block has varkeywords */
int ste_lineno; /* first line of block */
int ste_opt_lineno; /* lineno of last exec or import * */
int ste_tmpname; /* temporary name counter */
int ste_tmpname; /* counter for listcomp temp vars */
struct symtable *ste_table;
} PySymtableEntryObject;
} PySTEntryObject;
PyAPI_DATA(PyTypeObject) PySymtableEntry_Type;
PyAPI_DATA(PyTypeObject) PySTEntry_Type;
#define PySymtableEntry_Check(op) ((op)->ob_type == &PySymtableEntry_Type)
#define PySTEntry_Check(op) ((op)->ob_type == &PySTEntry_Type)
PyAPI_FUNC(PyObject *) PySymtableEntry_New(struct symtable *,
char *, int, int);
PyAPI_FUNC(PySTEntryObject *) \
PySTEntry_New(struct symtable *, identifier, block_ty, void *, int);
PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
PyFutureFeatures *);
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
PyAPI_FUNC(struct symtable *) PyNode_CompileSymtable(struct _node *, const char *);
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define TOP "global"
/* Flags for def-use information */
#define DEF_GLOBAL 1 /* global stmt */
@ -72,16 +67,19 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define DEF_STAR 2<<3 /* parameter is star arg */
#define DEF_DOUBLESTAR 2<<4 /* parameter is star-star arg */
#define DEF_INTUPLE 2<<5 /* name defined in tuple in parameters */
#define DEF_FREE 2<<6 /* name used but not defined in nested scope */
#define DEF_FREE 2<<6 /* name used but not defined in nested block */
#define DEF_FREE_GLOBAL 2<<7 /* free variable is actually implicit global */
#define DEF_FREE_CLASS 2<<8 /* free variable from class's method */
#define DEF_IMPORT 2<<9 /* assignment occurred via import */
#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
#define TYPE_FUNCTION 1
#define TYPE_CLASS 2
#define TYPE_MODULE 3
/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
table. GLOBAL is returned from PyST_GetScope() for either of them.
It is stored in ste_symbols at bits 12-14.
*/
#define SCOPE_OFF 11
#define SCOPE_MASK 7
#define LOCAL 1
#define GLOBAL_EXPLICIT 2
@ -89,9 +87,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define FREE 4
#define CELL 5
/* The following three names are used for the ste_unoptimized bit field */
#define OPT_IMPORT_STAR 1
#define OPT_EXEC 2
#define OPT_BARE_EXEC 4
#define OPT_TOPLEVEL 8 /* top-level names, including eval and exec */
#define GENERATOR 1
#define GENERATOR_EXPRESSION 2
#define GENERATOR 1
#define GENERATOR_EXPRESSION 2

1
Lib/cgitb.py
View File

@ -22,6 +22,7 @@ The default handler displays output as HTML.
"""
__author__ = 'Ka-Ping Yee'
__version__ = '$Revision$'
import sys

19
Lib/compiler/pyassem.py
View File

@ -364,16 +364,15 @@ class PyFlowGraph(FlowGraph):
def getCode(self):
"""Get a Python code object"""
if self.stage == RAW:
self.computeStackDepth()
self.flattenGraph()
if self.stage == FLAT:
self.convertArgs()
if self.stage == CONV:
self.makeByteCode()
if self.stage == DONE:
return self.newCodeObject()
raise RuntimeError, "inconsistent PyFlowGraph state"
assert self.stage == RAW
self.computeStackDepth()
self.flattenGraph()
assert self.stage == FLAT
self.convertArgs()
assert self.stage == CONV
self.makeByteCode()
assert self.stage == DONE
return self.newCodeObject()
def dump(self, io=None):
if io:

1
Lib/pydoc.py
View File

@ -36,6 +36,7 @@ Reference Manual pages.
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__date__ = "26 February 2001"
__version__ = "$Revision$"
__credits__ = """Guido van Rossum, for an excellent programming language.
Tommy Burnette, the original creator of manpy.

1
Lib/test/output/test_grammar
View File

@ -34,6 +34,7 @@ continue + try/except ok
continue + try/finally ok
testing continue and break in try/except in loop
return_stmt
yield_stmt
raise_stmt
import_name
import_from

2
Lib/test/output/test_profile
View File

@ -7,7 +7,7 @@ test_profile
12 0.000 0.000 0.012 0.001 :0(hasattr)
8 0.000 0.000 0.000 0.000 :0(range)
1 0.000 0.000 0.000 0.000 :0(setprofile)
1 0.000 0.000 1.000 1.000 <string>:1(?)
1 0.000 0.000 1.000 1.000 <string>:1(<module>)
0 0.000 0.000 profile:0(profiler)
1 0.000 0.000 1.000 1.000 profile:0(testfunc())
1 0.400 0.400 1.000 1.000 test_profile.py:23(testfunc)

85
Lib/test/test_code.py Normal file
View File

@ -0,0 +1,85 @@
"""This module includes tests of the code object representation.
>>> def f(x):
... def g(y):
... return x + y
... return g
...
>>> dump(f.func_code)
name: f
argcount: 1
names: ()
varnames: ('x', 'g')
cellvars: ('x',)
freevars: ()
nlocals: 2
flags: 3
consts: ('None', '<code object g>')
>>> dump(f(4).func_code)
name: g
argcount: 1
names: ()
varnames: ('y',)
cellvars: ()
freevars: ('x',)
nlocals: 1
flags: 19
consts: ('None',)
>>> def h(x, y):
... a = x + y
... b = x - y
... c = a * b
... return c
...
>>> dump(h.func_code)
name: h
argcount: 2
names: ()
varnames: ('x', 'y', 'a', 'b', 'c')
cellvars: ()
freevars: ()
nlocals: 5
flags: 67
consts: ('None',)
>>> def attrs(obj):
... print obj.attr1
... print obj.attr2
... print obj.attr3
>>> dump(attrs.func_code)
name: attrs
argcount: 1
names: ('attr1', 'attr2', 'attr3')
varnames: ('obj',)
cellvars: ()
freevars: ()
nlocals: 1
flags: 67
consts: ('None',)
"""
def consts(t):
"""Yield a doctest-safe sequence of object reprs."""
for elt in t:
r = repr(elt)
if r.startswith("<code object"):
yield "<code object %s>" % elt.co_name
else:
yield r
def dump(co):
"""Print out a text representation of a code object."""
for attr in ["name", "argcount", "names", "varnames", "cellvars",
"freevars", "nlocals", "flags"]:
print "%s: %s" % (attr, getattr(co, "co_" + attr))
print "consts:", tuple(consts(co.co_consts))
def test_main(verbose=None):
from test.test_support import run_doctest
from test import test_code
run_doctest(test_code, verbose)

12
Lib/test/test_doctest.py
View File

@ -1559,11 +1559,11 @@ Run the debugger on the docstring, and then restore sys.stdin.
>>> try: doctest.debug_src(s)
... finally: sys.stdin = real_stdin
> <string>(1)?()
> <string>(1)<module>()
(Pdb) next
12
--Return--
> <string>(1)?()->None
> <string>(1)<module>()->None
(Pdb) print x
12
(Pdb) continue
@ -1601,7 +1601,7 @@ def test_pdb_set_trace():
>>> try: runner.run(test)
... finally: sys.stdin = real_stdin
--Return--
> <doctest foo[1]>(1)?()->None
> <doctest foo[1]>(1)<module>()->None
-> import pdb; pdb.set_trace()
(Pdb) print x
42
@ -1637,7 +1637,7 @@ def test_pdb_set_trace():
(Pdb) print y
2
(Pdb) up
> <doctest foo[1]>(1)?()
> <doctest foo[1]>(1)<module>()
-> calls_set_trace()
(Pdb) print x
1
@ -1686,7 +1686,7 @@ def test_pdb_set_trace():
[EOF]
(Pdb) next
--Return--
> <doctest foo[2]>(1)?()->None
> <doctest foo[2]>(1)<module>()->None
-> f(3)
(Pdb) list
1 -> f(3)
@ -1779,7 +1779,7 @@ def test_pdb_set_trace_nested():
(Pdb) print y
1
(Pdb) up
> <doctest foo[1]>(1)?()
> <doctest foo[1]>(1)<module>()
-> calls_set_trace()
(Pdb) print foo
*** NameError: name 'foo' is not defined

8
Lib/test/test_eof.py
View File

@ -7,21 +7,21 @@ from test import test_support
class EOFTestCase(unittest.TestCase):
def test_EOFC(self):
expect = "EOL while scanning single-quoted string (<string>, line 1)"
try:
eval("""'this is a test\
""")
except SyntaxError, msg:
self.assertEqual(str(msg),
"EOL while scanning single-quoted string (line 1)")
self.assertEqual(str(msg), expect)
else:
raise test_support.TestFailed
def test_EOFS(self):
expect = "EOF while scanning triple-quoted string (<string>, line 1)"
try:
eval("""'''this is a test""")
except SyntaxError, msg:
self.assertEqual(str(msg),
"EOF while scanning triple-quoted string (line 1)")
self.assertEqual(str(msg), expect)
else:
raise test_support.TestFailed

2
Lib/test/test_generators.py
View File

@ -774,7 +774,7 @@ These are fine:
... try:
... 1//0
... except ZeroDivisionError:
... yield 666 # bad because *outer* try has finally
... yield 666
... except:
... pass
... finally:

5
Lib/test/test_genexps.py
View File

@ -125,13 +125,12 @@ Verify that syntax error's are raised for genexps used as lvalues
>>> (y for y in (1,2)) = 10
Traceback (most recent call last):
...
SyntaxError: assign to generator expression not possible
SyntaxError: assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[38]>, line 1)
>>> (y for y in (1,2)) += 10
Traceback (most recent call last):
...
SyntaxError: augmented assign to tuple literal, yield, or generator expression not possible
SyntaxError: augmented assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[39]>, line 1)
########### Tests borrowed from or inspired by test_generators.py ############

25
Lib/test/test_grammar.py
View File

@ -8,7 +8,7 @@
# regression test, the filterwarnings() call has been added to
# regrtest.py.
from test.test_support import TestFailed, verify, check_syntax
from test.test_support import TestFailed, verify, vereq, check_syntax
import sys
print '1. Parser'
@ -157,28 +157,31 @@ def f2(one_argument): pass
def f3(two, arguments): pass
def f4(two, (compound, (argument, list))): pass
def f5((compound, first), two): pass
verify(f2.func_code.co_varnames == ('one_argument',))
verify(f3.func_code.co_varnames == ('two', 'arguments'))
vereq(f2.func_code.co_varnames, ('one_argument',))
vereq(f3.func_code.co_varnames, ('two', 'arguments'))
if sys.platform.startswith('java'):
verify(f4.func_code.co_varnames ==
vereq(f4.func_code.co_varnames,
('two', '(compound, (argument, list))', 'compound', 'argument',
'list',))
verify(f5.func_code.co_varnames ==
vereq(f5.func_code.co_varnames,
('(compound, first)', 'two', 'compound', 'first'))
else:
verify(f4.func_code.co_varnames == ('two', '.2', 'compound',
'argument', 'list'))
verify(f5.func_code.co_varnames == ('.0', 'two', 'compound', 'first'))
vereq(f4.func_code.co_varnames,
('two', '.1', 'compound', 'argument', 'list'))
vereq(f5.func_code.co_varnames,
('.0', 'two', 'compound', 'first'))
def a1(one_arg,): pass
def a2(two, args,): pass
def v0(*rest): pass
def v1(a, *rest): pass
def v2(a, b, *rest): pass
def v3(a, (b, c), *rest): return a, b, c, rest
# ceval unpacks the formal arguments into the first argcount names;
# thus, the names nested inside tuples must appear after these names.
if sys.platform.startswith('java'):
verify(v3.func_code.co_varnames == ('a', '(b, c)', 'rest', 'b', 'c'))
else:
verify(v3.func_code.co_varnames == ('a', '.2', 'rest', 'b', 'c'))
vereq(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
verify(v3(1, (2, 3), 4) == (1, 2, 3, (4,)))
def d01(a=1): pass
d01()
@ -410,6 +413,10 @@ def g1(): return
def g2(): return 1
g1()
x = g2()
check_syntax("class foo:return 1")
print 'yield_stmt'
check_syntax("class foo:yield 1")
print 'raise_stmt' # 'raise' test [',' test]
try: raise RuntimeError, 'just testing'

13
Lib/test/test_import.py
View File

@ -192,3 +192,16 @@ def test_failing_reload():
del sys.modules[TESTFN]
test_failing_reload()
def test_import_name_binding():
# import x.y.z binds x in the current namespace
import test as x
import test.test_support
assert x is test, x.__name__
assert hasattr(test.test_support, "__file__")
# import x.y.z as w binds z as w
import test.test_support as y
assert y is test.test_support, y.__name__
test_import_name_binding()

24
Lib/test/test_parser.py
View File

@ -411,10 +411,32 @@ class IllegalSyntaxTestCase(unittest.TestCase):
(0, ''))
self.check_bad_tree(tree, "malformed global ast")
class CompileTestCase(unittest.TestCase):
# These tests are very minimal. :-(
def test_compile_expr(self):
st = parser.expr('2 + 3')
code = parser.compilest(st)
self.assertEquals(eval(code), 5)
def test_compile_suite(self):
st = parser.suite('x = 2; y = x + 3')
code = parser.compilest(st)
globs = {}
exec code in globs
self.assertEquals(globs['y'], 5)
def test_compile_error(self):
st = parser.suite('1 = 3 + 4')
self.assertRaises(SyntaxError, parser.compilest, st)
def test_main():
test_support.run_unittest(
RoundtripLegalSyntaxTestCase,
IllegalSyntaxTestCase
IllegalSyntaxTestCase,
CompileTestCase,
)

2
Lib/test/test_repr.py
View File

@ -123,7 +123,7 @@ class ReprTests(unittest.TestCase):
def test_lambda(self):
self.failUnless(repr(lambda x: x).startswith(
"<function <lambda"))
"<function lambda"))
# XXX anonymous functions? see func_repr
def test_builtin_function(self):

105
Lib/test/test_scope.py
View File

@ -1,4 +1,4 @@
from test.test_support import verify, TestFailed, check_syntax
from test.test_support import verify, TestFailed, check_syntax, vereq
import warnings
warnings.filterwarnings("ignore", r"import \*", SyntaxWarning, "<string>")
@ -13,8 +13,8 @@ def make_adder(x):
inc = make_adder(1)
plus10 = make_adder(10)
verify(inc(1) == 2)
verify(plus10(-2) == 8)
vereq(inc(1), 2)
vereq(plus10(-2), 8)
print "2. extra nesting"
@ -28,8 +28,8 @@ def make_adder2(x):
inc = make_adder2(1)
plus10 = make_adder2(10)
verify(inc(1) == 2)
verify(plus10(-2) == 8)
vereq(inc(1), 2)
vereq(plus10(-2), 8)
print "3. simple nesting + rebinding"
@ -42,8 +42,8 @@ def make_adder3(x):
inc = make_adder3(0)
plus10 = make_adder3(9)
verify(inc(1) == 2)
verify(plus10(-2) == 8)
vereq(inc(1), 2)
vereq(plus10(-2), 8)
print "4. nesting with global but no free"
@ -58,10 +58,10 @@ def make_adder4(): # XXX add exta level of indirection
global_x = 1
adder = make_adder4()
verify(adder(1) == 2)
vereq(adder(1), 2)
global_x = 10
verify(adder(-2) == 8)
vereq(adder(-2), 8)
print "5. nesting through class"
@ -74,8 +74,8 @@ def make_adder5(x):
inc = make_adder5(1)
plus10 = make_adder5(10)
verify(inc(1) == 2)
verify(plus10(-2) == 8)
vereq(inc(1), 2)
vereq(plus10(-2), 8)
print "6. nesting plus free ref to global"
@ -89,8 +89,8 @@ def make_adder6(x):
inc = make_adder6(1)
plus10 = make_adder6(10)
verify(inc(1) == 11) # there's only one global
verify(plus10(-2) == 8)
vereq(inc(1), 11) # there's only one global
vereq(plus10(-2), 8)
print "7. nearest enclosing scope"
@ -103,7 +103,7 @@ def f(x):
return g(2)
test_func = f(10)
verify(test_func(5) == 47)
vereq(test_func(5), 47)
print "8. mixed freevars and cellvars"
@ -123,7 +123,7 @@ def f(x, y, z):
g = f(1, 2, 3)
h = g(2, 4, 6)
verify(h() == 39)
vereq(h(), 39)
print "9. free variable in method"
@ -141,9 +141,9 @@ def test():
return Test()
t = test()
verify(t.test() == "var")
verify(t.method_and_var() == "method")
verify(t.actual_global() == "global")
vereq(t.test(), "var")
vereq(t.method_and_var(), "method")
vereq(t.actual_global(), "global")
method_and_var = "var"
class Test:
@ -158,9 +158,9 @@ class Test:
return str(self)
t = Test()
verify(t.test() == "var")
verify(t.method_and_var() == "method")
verify(t.actual_global() == "global")
vereq(t.test(), "var")
vereq(t.method_and_var(), "method")
vereq(t.actual_global(), "global")
print "10. recursion"
@ -175,7 +175,7 @@ def f(x):
else:
raise ValueError, "x must be >= 0"
verify(f(6) == 720)
vereq(f(6), 720)
print "11. unoptimized namespaces"
@ -252,24 +252,24 @@ print "12. lambdas"
f1 = lambda x: lambda y: x + y
inc = f1(1)
plus10 = f1(10)
verify(inc(1) == 2)
verify(plus10(5) == 15)
vereq(inc(1), 2)
vereq(plus10(5), 15)
f2 = lambda x: (lambda : lambda y: x + y)()
inc = f2(1)
plus10 = f2(10)
verify(inc(1) == 2)
verify(plus10(5) == 15)
vereq(inc(1), 2)
vereq(plus10(5), 15)
f3 = lambda x: lambda y: global_x + y
global_x = 1
inc = f3(None)
verify(inc(2) == 3)
vereq(inc(2), 3)
f8 = lambda x, y, z: lambda a, b, c: lambda : z * (b + y)
g = f8(1, 2, 3)
h = g(2, 4, 6)
verify(h() == 18)
vereq(h(), 18)
print "13. UnboundLocal"
@ -306,21 +306,21 @@ def makeReturner(*lst):
return lst
return returner
verify(makeReturner(1,2,3)() == (1,2,3))
vereq(makeReturner(1,2,3)(), (1,2,3))
def makeReturner2(**kwargs):
def returner():
return kwargs
return returner
verify(makeReturner2(a=11)()['a'] == 11)
vereq(makeReturner2(a=11)()['a'], 11)
def makeAddPair((a, b)):
def addPair((c, d)):
return (a + c, b + d)
return addPair
verify(makeAddPair((1, 2))((100, 200)) == (101,202))
vereq(makeAddPair((1, 2))((100, 200)), (101,202))
print "15. scope of global statements"
# Examples posted by Samuele Pedroni to python-dev on 3/1/2001
@ -337,8 +337,8 @@ def f():
return h()
return i()
return g()
verify(f() == 7)
verify(x == 7)
vereq(f(), 7)
vereq(x, 7)
# II
x = 7
@ -352,8 +352,8 @@ def f():
return h()
return i()
return g()
verify(f() == 2)
verify(x == 7)
vereq(f(), 2)
vereq(x, 7)
# III
x = 7
@ -368,8 +368,8 @@ def f():
return h()
return i()
return g()
verify(f() == 2)
verify(x == 2)
vereq(f(), 2)
vereq(x, 2)
# IV
x = 7
@ -384,8 +384,25 @@ def f():
return h()
return i()
return g()
verify(f() == 2)
verify(x == 2)
vereq(f(), 2)
vereq(x, 2)
# XXX what about global statements in class blocks?
# do they affect methods?
x = 12
class Global:
global x
x = 13
def set(self, val):
x = val
def get(self):
return x
g = Global()
vereq(g.get(), 13)
g.set(15)
vereq(g.get(), 13)
print "16. check leaks"
@ -407,7 +424,7 @@ def f1():
for i in range(100):
f1()
verify(Foo.count == 0)
vereq(Foo.count, 0)
print "17. class and global"
@ -419,9 +436,9 @@ def test(x):
return Foo()
x = 0
verify(test(6)(2) == 8)
vereq(test(6)(2), 8)
x = -1
verify(test(3)(2) == 5)
vereq(test(3)(2), 5)
print "18. verify that locals() works"
@ -437,7 +454,7 @@ def f(x):
d = f(2)(4)
verify(d.has_key('h'))
del d['h']
verify(d == {'x': 2, 'y': 7, 'w': 6})
vereq(d, {'x': 2, 'y': 7, 'w': 6})
print "19. var is bound and free in class"
@ -449,7 +466,7 @@ def f(x):
return C
inst = f(3)()
verify(inst.a == inst.m())
vereq(inst.a, inst.m())
print "20. interaction with trace function"

20
Makefile.pre.in
View File

@ -216,10 +216,22 @@ PGOBJS= \
PGENOBJS= $(PGENMAIN) $(POBJS) $(PGOBJS)
##########################################################################
# AST
AST_H= $(srcdir)/Include/Python-ast.h
AST_C= $(srcdir)/Python/Python-ast.c
AST_ASDL= $(srcdir)/Parser/Python.asdl
ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py
# XXX Note that a build now requires Python exist before the build starts
ASDLGEN= $(srcdir)/Parser/asdl_c.py -h $(srcdir)/Include -c $(srcdir)/Python
##########################################################################
# Python
PYTHON_OBJS= \
Python/Python-ast.o \
Python/asdl.o \
Python/ast.o \
Python/bltinmodule.o \
Python/exceptions.o \
Python/ceval.o \
@ -265,6 +277,7 @@ OBJECT_OBJS= \
Objects/cellobject.o \
Objects/classobject.o \
Objects/cobject.o \
Objects/codeobject.o \
Objects/complexobject.o \
Objects/descrobject.o \
Objects/enumobject.o \
@ -457,8 +470,10 @@ Parser/metagrammar.o: $(srcdir)/Parser/metagrammar.c
Parser/tokenizer_pgen.o: $(srcdir)/Parser/tokenizer.c
$(AST_H) $(AST_C): $(AST_ASDL) $(ASDLGEN_FILES)
$(ASDLGEN) $(AST_ASDL)
Python/compile.o Python/symtable.o: $(GRAMMAR_H)
Python/compile.o Python/symtable.o: $(GRAMMAR_H) $(AST_H)
Python/getplatform.o: $(srcdir)/Python/getplatform.c
$(CC) -c $(PY_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@ -474,12 +489,15 @@ Objects/unicodectype.o: $(srcdir)/Objects/unicodectype.c \
PYTHON_HEADERS= \
Include/Python.h \
Include/Python-ast.h \
Include/asdl.h \
Include/abstract.h \
Include/boolobject.h \
Include/bufferobject.h \
Include/ceval.h \
Include/classobject.h \
Include/cobject.h \
Include/code.h \
Include/codecs.h \
Include/compile.h \
Include/complexobject.h \

1
Misc/ACKS
View File

@ -165,6 +165,7 @@ Eugene Dvurechenski
Maxim Dzumanenko
Hans Eckardt
Grant Edwards
John Ehresman
Andrew Eland
Lance Ellinghaus
David Ely

1
Modules/_hotshot.c
View File

@ -3,6 +3,7 @@
*/
#include "Python.h"
#include "code.h"
#include "compile.h"
#include "eval.h"
#include "frameobject.h"

8
Modules/symtablemodule.c
View File

@ -1,6 +1,8 @@
#include "Python.h"
#include "code.h"
#include "compile.h"
#include "Python-ast.h"
#include "symtable.h"
static PyObject *
@ -64,9 +66,9 @@ init_symtable(void)
PyModule_AddIntConstant(m, "DEF_IMPORT", DEF_IMPORT);
PyModule_AddIntConstant(m, "DEF_BOUND", DEF_BOUND);
PyModule_AddIntConstant(m, "TYPE_FUNCTION", TYPE_FUNCTION);
PyModule_AddIntConstant(m, "TYPE_CLASS", TYPE_CLASS);
PyModule_AddIntConstant(m, "TYPE_MODULE", TYPE_MODULE);
PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock);
PyModule_AddIntConstant(m, "TYPE_CLASS", ClassBlock);
PyModule_AddIntConstant(m, "TYPE_MODULE", ModuleBlock);
PyModule_AddIntConstant(m, "OPT_IMPORT_STAR", OPT_IMPORT_STAR);
PyModule_AddIntConstant(m, "OPT_EXEC", OPT_EXEC);

453
Objects/codeobject.c Normal file
View File

@ -0,0 +1,453 @@
#include "Python.h"
#include "code.h"
#include "structmember.h"
#define NAME_CHARS \
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
/* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */
static int
all_name_chars(unsigned char *s)
{
static char ok_name_char[256];
static unsigned char *name_chars = (unsigned char *)NAME_CHARS;
if (ok_name_char[*name_chars] == 0) {
unsigned char *p;
for (p = name_chars; *p; p++)
ok_name_char[*p] = 1;
}
while (*s) {
if (ok_name_char[*s++] == 0)
return 0;
}
return 1;
}
static void
intern_strings(PyObject *tuple)
{
int i;
for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
PyObject *v = PyTuple_GET_ITEM(tuple, i);
if (v == NULL || !PyString_CheckExact(v)) {
Py_FatalError("non-string found in code slot");
}
PyString_InternInPlace(&PyTuple_GET_ITEM(tuple, i));
}
}
PyCodeObject *
PyCode_New(int argcount, int nlocals, int stacksize, int flags,
PyObject *code, PyObject *consts, PyObject *names,
PyObject *varnames, PyObject *freevars, PyObject *cellvars,
PyObject *filename, PyObject *name, int firstlineno,
PyObject *lnotab)
{
PyCodeObject *co;
int i;
/* Check argument types */
if (argcount < 0 || nlocals < 0 ||
code == NULL ||
consts == NULL || !PyTuple_Check(consts) ||
names == NULL || !PyTuple_Check(names) ||
varnames == NULL || !PyTuple_Check(varnames) ||
freevars == NULL || !PyTuple_Check(freevars) ||
cellvars == NULL || !PyTuple_Check(cellvars) ||
name == NULL || !PyString_Check(name) ||
filename == NULL || !PyString_Check(filename) ||
lnotab == NULL || !PyString_Check(lnotab) ||
!PyObject_CheckReadBuffer(code)) {
PyErr_BadInternalCall();
return NULL;
}
intern_strings(names);
intern_strings(varnames);
intern_strings(freevars);
intern_strings(cellvars);
/* Intern selected string constants */
for (i = PyTuple_Size(consts); --i >= 0; ) {
PyObject *v = PyTuple_GetItem(consts, i);
if (!PyString_Check(v))
continue;
if (!all_name_chars((unsigned char *)PyString_AS_STRING(v)))
continue;
PyString_InternInPlace(&PyTuple_GET_ITEM(consts, i));
}
co = PyObject_NEW(PyCodeObject, &PyCode_Type);
if (co != NULL) {
co->co_argcount = argcount;
co->co_nlocals = nlocals;
co->co_stacksize = stacksize;
co->co_flags = flags;
Py_INCREF(code);
co->co_code = code;
Py_INCREF(consts);
co->co_consts = consts;
Py_INCREF(names);
co->co_names = names;
Py_INCREF(varnames);
co->co_varnames = varnames;
Py_INCREF(freevars);
co->co_freevars = freevars;
Py_INCREF(cellvars);
co->co_cellvars = cellvars;
Py_INCREF(filename);
co->co_filename = filename;
Py_INCREF(name);
co->co_name = name;
co->co_firstlineno = firstlineno;
Py_INCREF(lnotab);
co->co_lnotab = lnotab;
}
return co;
}
#define OFF(x) offsetof(PyCodeObject, x)
static PyMemberDef code_memberlist[] = {
{"co_argcount", T_INT, OFF(co_argcount), READONLY},
{"co_nlocals", T_INT, OFF(co_nlocals), READONLY},
{"co_stacksize",T_INT, OFF(co_stacksize), READONLY},
{"co_flags", T_INT, OFF(co_flags), READONLY},
{"co_code", T_OBJECT, OFF(co_code), READONLY},
{"co_consts", T_OBJECT, OFF(co_consts), READONLY},
{"co_names", T_OBJECT, OFF(co_names), READONLY},
{"co_varnames", T_OBJECT, OFF(co_varnames), READONLY},
{"co_freevars", T_OBJECT, OFF(co_freevars), READONLY},
{"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY},
{"co_filename", T_OBJECT, OFF(co_filename), READONLY},
{"co_name", T_OBJECT, OFF(co_name), READONLY},
{"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
{"co_lnotab", T_OBJECT, OFF(co_lnotab), READONLY},
{NULL} /* Sentinel */
};
/* Helper for code_new: return a shallow copy of a tuple that is
guaranteed to contain exact strings, by converting string subclasses
to exact strings and complaining if a non-string is found. */
static PyObject*
validate_and_copy_tuple(PyObject *tup)
{
PyObject *newtuple;
PyObject *item;
int i, len;
len = PyTuple_GET_SIZE(tup);
newtuple = PyTuple_New(len);
if (newtuple == NULL)
return NULL;
for (i = 0; i < len; i++) {
item = PyTuple_GET_ITEM(tup, i);
if (PyString_CheckExact(item)) {
Py_INCREF(item);
}
else if (!PyString_Check(item)) {
PyErr_Format(
PyExc_TypeError,
"name tuples must contain only "
"strings, not '%.500s'",
item->ob_type->tp_name);
Py_DECREF(newtuple);
return NULL;
}
else {
item = PyString_FromStringAndSize(
PyString_AS_STRING(item),
PyString_GET_SIZE(item));
if (item == NULL) {
Py_DECREF(newtuple);
return NULL;
}
}
PyTuple_SET_ITEM(newtuple, i, item);
}
return newtuple;
}
PyDoc_STRVAR(code_doc,
"code(argcount, nlocals, stacksize, flags, codestring, constants, names,\n\
varnames, filename, name, firstlineno, lnotab[, freevars[, cellvars]])\n\
\n\
Create a code object. Not for the faint of heart.");
static PyObject *
code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
int argcount;
int nlocals;
int stacksize;
int flags;
PyObject *co = NULL;
PyObject *code;
PyObject *consts;
PyObject *names, *ournames = NULL;
PyObject *varnames, *ourvarnames = NULL;
PyObject *freevars = NULL, *ourfreevars = NULL;
PyObject *cellvars = NULL, *ourcellvars = NULL;
PyObject *filename;
PyObject *name;
int firstlineno;
PyObject *lnotab;
if (!PyArg_ParseTuple(args, "iiiiSO!O!O!SSiS|O!O!:code",
&argcount, &nlocals, &stacksize, &flags,
&code,
&PyTuple_Type, &consts,
&PyTuple_Type, &names,
&PyTuple_Type, &varnames,
&filename, &name,
&firstlineno, &lnotab,
&PyTuple_Type, &freevars,
&PyTuple_Type, &cellvars))
return NULL;
if (argcount < 0) {
PyErr_SetString(
PyExc_ValueError,
"code: argcount must not be negative");
goto cleanup;
}
if (nlocals < 0) {
PyErr_SetString(
PyExc_ValueError,
"code: nlocals must not be negative");
goto cleanup;
}
ournames = validate_and_copy_tuple(names);
if (ournames == NULL)
goto cleanup;
ourvarnames = validate_and_copy_tuple(varnames);
if (ourvarnames == NULL)
goto cleanup;
if (freevars)
ourfreevars = validate_and_copy_tuple(freevars);
else
ourfreevars = PyTuple_New(0);
if (ourfreevars == NULL)
goto cleanup;
if (cellvars)
ourcellvars = validate_and_copy_tuple(cellvars);
else
ourcellvars = PyTuple_New(0);
if (ourcellvars == NULL)
goto cleanup;
co = (PyObject *)PyCode_New(argcount, nlocals, stacksize, flags,
code, consts, ournames, ourvarnames,
ourfreevars, ourcellvars, filename,
name, firstlineno, lnotab);
cleanup:
Py_XDECREF(ournames);
Py_XDECREF(ourvarnames);
Py_XDECREF(ourfreevars);
Py_XDECREF(ourcellvars);
return co;
}
static void
code_dealloc(PyCodeObject *co)
{
Py_XDECREF(co->co_code);
Py_XDECREF(co->co_consts);
Py_XDECREF(co->co_names);
Py_XDECREF(co->co_varnames);
Py_XDECREF(co->co_freevars);
Py_XDECREF(co->co_cellvars);
Py_XDECREF(co->co_filename);
Py_XDECREF(co->co_name);
Py_XDECREF(co->co_lnotab);
PyObject_DEL(co);
}
static PyObject *
code_repr(PyCodeObject *co)
{
char buf[500];
int lineno = -1;
char *filename = "???";
char *name = "???";
if (co->co_firstlineno != 0)
lineno = co->co_firstlineno;
if (co->co_filename && PyString_Check(co->co_filename))
filename = PyString_AS_STRING(co->co_filename);
if (co->co_name && PyString_Check(co->co_name))
name = PyString_AS_STRING(co->co_name);
PyOS_snprintf(buf, sizeof(buf),
"<code object %.100s at %p, file \"%.300s\", line %d>",
name, co, filename, lineno);
return PyString_FromString(buf);
}
static int
code_compare(PyCodeObject *co, PyCodeObject *cp)
{
int cmp;
cmp = PyObject_Compare(co->co_name, cp->co_name);
if (cmp) return cmp;
cmp = co->co_argcount - cp->co_argcount;
if (cmp) goto normalize;
cmp = co->co_nlocals - cp->co_nlocals;
if (cmp) goto normalize;
cmp = co->co_flags - cp->co_flags;
if (cmp) goto normalize;
cmp = co->co_firstlineno - cp->co_firstlineno;
if (cmp) goto normalize;
cmp = PyObject_Compare(co->co_code, cp->co_code);
if (cmp) return cmp;
cmp = PyObject_Compare(co->co_consts, cp->co_consts);
if (cmp) return cmp;
cmp = PyObject_Compare(co->co_names, cp->co_names);
if (cmp) return cmp;
cmp = PyObject_Compare(co->co_varnames, cp->co_varnames);
if (cmp) return cmp;
cmp = PyObject_Compare(co->co_freevars, cp->co_freevars);
if (cmp) return cmp;
cmp = PyObject_Compare(co->co_cellvars, cp->co_cellvars);
return cmp;
normalize:
if (cmp > 0)
return 1;
else if (cmp < 0)
return -1;
else
return 0;
}
static long
code_hash(PyCodeObject *co)
{
long h, h0, h1, h2, h3, h4, h5, h6;
h0 = PyObject_Hash(co->co_name);
if (h0 == -1) return -1;
h1 = PyObject_Hash(co->co_code);
if (h1 == -1) return -1;
h2 = PyObject_Hash(co->co_consts);
if (h2 == -1) return -1;
h3 = PyObject_Hash(co->co_names);
if (h3 == -1) return -1;
h4 = PyObject_Hash(co->co_varnames);
if (h4 == -1) return -1;
h5 = PyObject_Hash(co->co_freevars);
if (h5 == -1) return -1;
h6 = PyObject_Hash(co->co_cellvars);
if (h6 == -1) return -1;
h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^
co->co_argcount ^ co->co_nlocals ^ co->co_flags;
if (h == -1) h = -2;
return h;
}
/* XXX code objects need to participate in GC? */
PyTypeObject PyCode_Type = {
PyObject_HEAD_INIT(&PyType_Type)
0,
"code",
sizeof(PyCodeObject),
0,
(destructor)code_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
(cmpfunc)code_compare, /* tp_compare */
(reprfunc)code_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
(hashfunc)code_hash, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
code_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
code_memberlist, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
code_new, /* tp_new */
};
/* All about c_lnotab.
c_lnotab is an array of unsigned bytes disguised as a Python string. In -O
mode, SET_LINENO opcodes aren't generated, and bytecode offsets are mapped
to source code line #s (when needed for tracebacks) via c_lnotab instead.
The array is conceptually a list of
(bytecode offset increment, line number increment)
pairs. The details are important and delicate, best illustrated by example:
byte code offset source code line number
0 1
6 2
50 7
350 307
361 308
The first trick is that these numbers aren't stored, only the increments
from one row to the next (this doesn't really work, but it's a start):
0, 1, 6, 1, 44, 5, 300, 300, 11, 1
The second trick is that an unsigned byte can't hold negative values, or
values larger than 255, so (a) there's a deep assumption that byte code
offsets and their corresponding line #s both increase monotonically, and (b)
if at least one column jumps by more than 255 from one row to the next, more
than one pair is written to the table. In case #b, there's no way to know
from looking at the table later how many were written. That's the delicate
part. A user of c_lnotab desiring to find the source line number
corresponding to a bytecode address A should do something like this
lineno = addr = 0
for addr_incr, line_incr in c_lnotab:
addr += addr_incr
if addr > A:
return lineno
lineno += line_incr
In order for this to work, when the addr field increments by more than 255,
the line # increment in each pair generated must be 0 until the remaining addr
increment is < 256. So, in the example above, com_set_lineno should not (as
was actually done until 2.2) expand 300, 300 to 255, 255, 45, 45, but to
255, 0, 45, 255, 0, 45.
*/
int
PyCode_Addr2Line(PyCodeObject *co, int addrq)
{
int size = PyString_Size(co->co_lnotab) / 2;
unsigned char *p = (unsigned char*)PyString_AsString(co->co_lnotab);
int line = co->co_firstlineno;
int addr = 0;
while (--size >= 0) {
addr += *p++;
if (addr > addrq)
break;
line += *p++;
}
return line;
}

1
Objects/frameobject.c
View File

@ -3,6 +3,7 @@
#include "Python.h"
#include "code.h"
#include "compile.h"
#include "frameobject.h"
#include "opcode.h"

6
Objects/funcobject.c
View File

@ -2,7 +2,7 @@
/* Function object implementation */
#include "Python.h"
#include "compile.h"
#include "code.h"
#include "eval.h"
#include "structmember.h"
@ -144,7 +144,9 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure)
Py_XINCREF(closure);
}
else {
PyErr_SetString(PyExc_SystemError, "non-tuple closure");
PyErr_Format(PyExc_SystemError,
"expected tuple for closure, got '%.100s'",
closure->ob_type->tp_name);
return -1;
}
Py_XDECREF(((PyFunctionObject *) op) -> func_closure);

12
Objects/typeobject.c
View File

@ -1737,20 +1737,14 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
goto bad_slots;
for (i = j = 0; i < nslots; i++) {
char *s;
char buffer[256];
tmp = PyTuple_GET_ITEM(slots, i);
s = PyString_AS_STRING(tmp);
if ((add_dict && strcmp(s, "__dict__") == 0) ||
(add_weak && strcmp(s, "__weakref__") == 0))
continue;
if (_Py_Mangle(PyString_AS_STRING(name),
PyString_AS_STRING(tmp),
buffer, sizeof(buffer)))
{
tmp = PyString_FromString(buffer);
} else {
Py_INCREF(tmp);
}
tmp =_Py_Mangle(name, tmp);
if (!tmp)
goto bad_slots;
PyTuple_SET_ITEM(newslots, j, tmp);
j++;
}

164
PCbuild/pythoncore.vcproj
View File

@ -3,6 +3,7 @@
ProjectType="Visual C++"
Version="7.10"
Name="pythoncore"
ProjectGUID="{7AFA1F0B-A8A1-455A-A832-BF263404BBEF}"
SccProjectName="pythoncore"
SccLocalPath="..">
<Platforms>
@ -477,6 +478,12 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Python\asdl.c">
</File>
<File
RelativePath="..\Python\ast.c">
</File>
<File
RelativePath="..\Modules\audioop.c">
<FileConfiguration
@ -802,34 +809,10 @@
</FileConfiguration>
</File>
<File
RelativePath="..\Modules\collectionsmodule.c">
RelativePath="..\Objects\codeobject.c">
</File>
<File
RelativePath="..\Python\compile.c">
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="ReleaseItanium|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
RelativePath="..\Modules\collectionsmodule.c">
</File>
<File
RelativePath="..\Objects\complexobject.c">
@ -1185,6 +1168,9 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Parser\firstsets.c">
</File>
<File
RelativePath="..\Objects\floatobject.c">
<FileConfiguration
@ -1320,6 +1306,33 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Modules\functionalmodule.c">
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="ReleaseItanium|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Python\future.c">
<FileConfiguration
@ -1647,6 +1660,9 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Parser\grammar.c">
</File>
<File
RelativePath="..\Parser\grammar1.c">
<FileConfiguration
@ -2301,6 +2317,9 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Python\compile.c">
</File>
<File
RelativePath="..\Parser\node.c">
<FileConfiguration
@ -2436,9 +2455,6 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Modules\parsermodule.c">
</File>
<File
RelativePath="..\Parser\parsetok.c">
<FileConfiguration
@ -2466,6 +2482,9 @@
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Parser\pgen.c">
</File>
<File
RelativePath="..\Modules\posixmodule.c">
<FileConfiguration
@ -2550,6 +2569,9 @@
<File
RelativePath="..\Python\pystrtod.c">
</File>
<File
RelativePath="..\Python\Python-ast.c">
</File>
<File
RelativePath="..\PC\python_nt.rc">
<FileConfiguration
@ -2764,6 +2786,7 @@
</FileConfiguration>
</File>
<File
<<<<<<< pythoncore.vcproj
RelativePath="..\Modules\sha256module.c">
<FileConfiguration
Name="Release|Win32">
@ -2819,6 +2842,63 @@
</File>
<File
RelativePath="..\Modules\signalmodule.c">
=======
RelativePath="..\Modules\sha256module.c">
>>>>>>> 1.26.2.3
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="ReleaseItanium|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Modules\sha512module.c">
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="ReleaseItanium|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
</File>
<File
RelativePath="..\Modules\signalmodule.c">
<FileConfiguration
Name="Release|Win32">
<Tool
@ -3145,31 +3225,7 @@
</FileConfiguration>
</File>
<File
RelativePath="..\Parser\tokenizer.c">
<FileConfiguration
Name="Release|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="Debug|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="_DEBUG;USE_DL_EXPORT;WIN32;_WINDOWS;$(NoInherit)"/>
</FileConfiguration>
<FileConfiguration
Name="ReleaseItanium|Win32">
<Tool
Name="VCCLCompilerTool"
Optimization="2"
AdditionalIncludeDirectories=""
PreprocessorDefinitions="NDEBUG;WIN32;_WINDOWS;USE_DL_EXPORT;$(NoInherit)"/>
</FileConfiguration>
RelativePath="..\Parser\tokenizer_pgen.c">
</File>
<File
RelativePath="..\Python\traceback.c">

3
Parser/.cvsignore
View File

@ -1,3 +1,6 @@
Makefile
pgen
add2lib
asdl.pyc
asdl_c.pyc
spark.pyc

107
Parser/Python.asdl Normal file
View File

@ -0,0 +1,107 @@
-- ASDL's three builtin types are identifier, int, string
module Python
{
mod = Module(stmt* body)
| Interactive(stmt* body)
| Expression(expr body)
-- not really an actual node but useful in Jython's typesystem.
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorators)
| ClassDef(identifier name, expr* bases, stmt* body)
| Return(expr? value)
| Delete(expr* targets)
| Assign(expr* targets, expr value)
| AugAssign(expr target, operator op, expr value)
-- not sure if bool is allowed, can always use int
| Print(expr? dest, expr* values, bool nl)
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
-- 'type' is a bad name
| Raise(expr? type, expr? inst, expr? tback)
| TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
| TryFinally(stmt* body, stmt* finalbody)
| Assert(expr test, expr? msg)
| Import(alias* names)
| ImportFrom(identifier module, alias* names)
-- Doesn't capture requirement that locals must be
-- defined if globals is
-- still supports use as a function!
| Exec(expr body, expr? globals, expr? locals)
| Global(identifier* names)
| Expr(expr value)
| Pass | Break | Continue
-- XXX Jython will be different
attributes (int lineno)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
| BinOp(expr left, operator op, expr right)
| UnaryOp(unaryop op, expr operand)
| Lambda(arguments args, expr body)
| Dict(expr* keys, expr* values)
| ListComp(expr elt, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
| Yield(expr? value)
-- need sequences for compare to distinguish between
-- x < 4 < 3 and (x < 4) < 3
| Compare(expr left, cmpop* ops, expr* comparators)
| Call(expr func, expr* args, keyword* keywords,
expr? starargs, expr? kwargs)
| Repr(expr value)
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
-- other literals? bools?
-- the following expression can appear in assignment context
| Attribute(expr value, identifier attr, expr_context ctx)
| Subscript(expr value, slice slice, expr_context ctx)
| Name(identifier id, expr_context ctx)
| List(expr* elts, expr_context ctx)
| Tuple(expr *elts, expr_context ctx)
attributes (int lineno)
expr_context = Load | Store | Del | AugLoad | AugStore | Param
slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step)
| ExtSlice(slice* dims)
| Index(expr value)
boolop = And | Or
operator = Add | Sub | Mult | Div | Mod | Pow | LShift
| RShift | BitOr | BitXor | BitAnd | FloorDiv
unaryop = Invert | Not | UAdd | USub
cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
comprehension = (expr target, expr iter, expr* ifs)
-- not sure what to call the first argument for raise and except
excepthandler = (expr? type, expr? name, stmt* body)
arguments = (expr* args, identifier? vararg,
identifier? kwarg, expr* defaults)
-- keyword arguments supplied to call
keyword = (identifier arg, expr value)
-- import name with optional 'as' alias.
alias = (identifier name, identifier? asname)
}

393
Parser/asdl.py Normal file
View File

@ -0,0 +1,393 @@
"""An implementation of the Zephyr Abstract Syntax Definition Language.
See http://asdl.sourceforge.net/ and
http://www.cs.princeton.edu/~danwang/Papers/dsl97/dsl97-abstract.html.
Only supports top level module decl, not view. I'm guessing that view
is intended to support the browser and I'm not interested in the
browser.
"""
#__metaclass__ = type
import os
import traceback
import spark
class Token:
# spark seems to dispatch in the parser based on a token's
# type attribute
def __init__(self, type, lineno):
self.type = type
self.lineno = lineno
def __str__(self):
return self.type
def __repr__(self):
return str(self)
class Id(Token):
def __init__(self, value, lineno):
self.type = 'Id'
self.value = value
self.lineno = lineno
def __str__(self):
return self.value
class ASDLSyntaxError:
def __init__(self, lineno, token=None, msg=None):
self.lineno = lineno
self.token = token
self.msg = msg
def __str__(self):
if self.msg is None:
return "Error at '%s', line %d" % (self.token, self.lineno)
else:
return "%s, line %d" % (self.msg, self.lineno)
class ASDLScanner(spark.GenericScanner, object):
def tokenize(self, input):
self.rv = []
self.lineno = 1
super(ASDLScanner, self).tokenize(input)
return self.rv
def t_id(self, s):
r"[\w\.]+"
# XXX doesn't distinguish upper vs. lower, which is
# significant for ASDL.
self.rv.append(Id(s, self.lineno))
def t_xxx(self, s): # not sure what this production means
r"<="
self.rv.append(Token(s, self.lineno))
def t_punctuation(self, s):
r"[\{\}\*\=\|\(\)\,\?\:]"
self.rv.append(Token(s, self.lineno))
def t_comment(self, s):
r"\-\-[^\n]*"
pass
def t_newline(self, s):
r"\n"
self.lineno += 1
def t_whitespace(self, s):
r"[ \t]+"
pass
def t_default(self, s):
r" . +"
raise ValueError, "unmatched input: %s" % `s`
class ASDLParser(spark.GenericParser, object):
def __init__(self):
super(ASDLParser, self).__init__("module")
def typestring(self, tok):
return tok.type
def error(self, tok):
raise ASDLSyntaxError(tok.lineno, tok)
def p_module_0(self, (module, name, _0, _1)):
" module ::= Id Id { } "
if module.value != "module":
raise ASDLSyntaxError(module.lineno,
msg="expected 'module', found %s" % module)
return Module(name, None)
def p_module(self, (module, name, _0, definitions, _1)):
" module ::= Id Id { definitions } "
if module.value != "module":
raise ASDLSyntaxError(module.lineno,
msg="expected 'module', found %s" % module)
return Module(name, definitions)
def p_definition_0(self, (definition,)):
" definitions ::= definition "
return definition
def p_definition_1(self, (definitions, definition)):
" definitions ::= definition definitions "
return definitions + definition
def p_definition(self, (id, _, type)):
" definition ::= Id = type "
return [Type(id, type)]
def p_type_0(self, (product,)):
" type ::= product "
return product
def p_type_1(self, (sum,)):
" type ::= sum "
return Sum(sum)
def p_type_2(self, (sum, id, _0, attributes, _1)):
" type ::= sum Id ( fields ) "
if id.value != "attributes":
raise ASDLSyntaxError(id.lineno,
msg="expected attributes, found %s" % id)
return Sum(sum, attributes)
def p_product(self, (_0, fields, _1)):
" product ::= ( fields ) "
# XXX can't I just construct things in the right order?
fields.reverse()
return Product(fields)
def p_sum_0(self, (constructor,)):
" sum ::= constructor """
return [constructor]
def p_sum_1(self, (constructor, _, sum)):
" sum ::= constructor | sum "
return [constructor] + sum
def p_sum_2(self, (constructor, _, sum)):
" sum ::= constructor | sum "
return [constructor] + sum
def p_constructor_0(self, (id,)):
" constructor ::= Id "
return Constructor(id)
def p_constructor_1(self, (id, _0, fields, _1)):
" constructor ::= Id ( fields ) "
# XXX can't I just construct things in the right order?
fields.reverse()
return Constructor(id, fields)
def p_fields_0(self, (field,)):
" fields ::= field "
return [field]
def p_fields_1(self, (field, _, fields)):
" fields ::= field , fields "
return fields + [field]
def p_field_0(self, (type,)):
" field ::= Id "
return Field(type)
def p_field_1(self, (type, name)):
" field ::= Id Id "
return Field(type, name)
def p_field_2(self, (type, _, name)):
" field ::= Id * Id "
return Field(type, name, seq=1)
def p_field_3(self, (type, _, name)):
" field ::= Id ? Id "
return Field(type, name, opt=1)
def p_field_4(self, (type, _)):
" field ::= Id * "
return Field(type, seq=1)
def p_field_5(self, (type, _)):
" field ::= Id ? "
return Field(type, opt=1)
builtin_types = ("identifier", "string", "int", "bool", "object")
# below is a collection of classes to capture the AST of an AST :-)
# not sure if any of the methods are useful yet, but I'm adding them
# piecemeal as they seem helpful
class AST:
pass # a marker class
class Module(AST):
def __init__(self, name, dfns):
self.name = name
self.dfns = dfns
self.types = {} # maps type name to value (from dfns)
for type in dfns:
self.types[type.name.value] = type.value
def __repr__(self):
return "Module(%s, %s)" % (self.name, self.dfns)
class Type(AST):
def __init__(self, name, value):
self.name = name
self.value = value
def __repr__(self):
return "Type(%s, %s)" % (self.name, self.value)
class Constructor(AST):
def __init__(self, name, fields=None):
self.name = name
self.fields = fields or []
def __repr__(self):
return "Constructor(%s, %s)" % (self.name, self.fields)
class Field(AST):
def __init__(self, type, name=None, seq=0, opt=0):
self.type = type
self.name = name
self.seq = seq
self.opt = opt
def __repr__(self):
if self.seq:
extra = ", seq=1"
elif self.opt:
extra = ", opt=1"
else:
extra = ""
if self.name is None:
return "Field(%s%s)" % (self.type, extra)
else:
return "Field(%s, %s%s)" % (self.type, self.name, extra)
class Sum(AST):
def __init__(self, types, attributes=None):
self.types = types
self.attributes = attributes or []
def __repr__(self):
if self.attributes is None:
return "Sum(%s)" % self.types
else:
return "Sum(%s, %s)" % (self.types, self.attributes)
class Product(AST):
def __init__(self, fields):
self.fields = fields
def __repr__(self):
return "Product(%s)" % self.fields
class VisitorBase(object):
def __init__(self, skip=0):
self.cache = {}
self.skip = skip
def visit(self, object, *args):
meth = self._dispatch(object)
if meth is None:
return
try:
meth(object, *args)
except Exception, err:
print "Error visiting", repr(object)
print err
traceback.print_exc()
# XXX hack
if hasattr(self, 'file'):
self.file.flush()
os._exit(1)
def _dispatch(self, object):
assert isinstance(object, AST), repr(object)
klass = object.__class__
meth = self.cache.get(klass)
if meth is None:
methname = "visit" + klass.__name__
if self.skip:
meth = getattr(self, methname, None)
else:
meth = getattr(self, methname)
self.cache[klass] = meth
return meth
class Check(VisitorBase):
def __init__(self):
super(Check, self).__init__(skip=1)
self.cons = {}
self.errors = 0
self.types = {}
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, str(type.name))
def visitSum(self, sum, name):
for t in sum.types:
self.visit(t, name)
def visitConstructor(self, cons, name):
key = str(cons.name)
conflict = self.cons.get(key)
if conflict is None:
self.cons[key] = name
else:
print "Redefinition of constructor %s" % key
print "Defined in %s and %s" % (conflict, name)
self.errors += 1
for f in cons.fields:
self.visit(f, key)
def visitField(self, field, name):
key = str(field.type)
l = self.types.setdefault(key, [])
l.append(name)
def visitProduct(self, prod, name):
for f in prod.fields:
self.visit(f, name)
def check(mod):
v = Check()
v.visit(mod)
for t in v.types:
if not mod.types.has_key(t) and not t in builtin_types:
v.errors += 1
uses = ", ".join(v.types[t])
print "Undefined type %s, used in %s" % (t, uses)
return not v.errors
def parse(file):
scanner = ASDLScanner()
parser = ASDLParser()
buf = open(file).read()
tokens = scanner.tokenize(buf)
try:
return parser.parse(tokens)
except ASDLSyntaxError, err:
print err
lines = buf.split("\n")
print lines[err.lineno - 1] # lines starts at 0, files at 1
if __name__ == "__main__":
import glob
import sys
if len(sys.argv) > 1:
files = sys.argv[1:]
else:
testdir = "tests"
files = glob.glob(testdir + "/*.asdl")
for file in files:
print file
mod = parse(file)
print "module", mod.name
print len(mod.dfns), "definitions"
if not check(mod):
print "Check failed"
else:
for dfn in mod.dfns:
print dfn.type

621
Parser/asdl_c.py Executable file
View File

@ -0,0 +1,621 @@
#! /usr/bin/env python
"""Generate C code from an ASDL description."""
# TO DO
# handle fields that have a type but no name
import os, sys, traceback
import asdl
TABSIZE = 8
MAX_COL = 80
def get_c_type(name):
"""Return a string for the C name of the type.
This function special cases the default types provided by asdl:
identifier, string, int, bool.
"""
# XXX ack! need to figure out where Id is useful and where string
if isinstance(name, asdl.Id):
name = name.value
if name in asdl.builtin_types:
return name
else:
return "%s_ty" % name
def reflow_lines(s, depth):
"""Reflow the line s indented depth tabs.
Return a sequence of lines where no line extends beyond MAX_COL
when properly indented. The first line is properly indented based
exclusively on depth * TABSIZE. All following lines -- these are
the reflowed lines generated by this function -- start at the same
column as the first character beyond the opening { in the first
line.
"""
size = MAX_COL - depth * TABSIZE
if len(s) < size:
return [s]
lines = []
cur = s
padding = ""
while len(cur) > size:
i = cur.rfind(' ', 0, size)
# XXX this should be fixed for real
if i == -1 and 'GeneratorExp' in cur:
i = size + 3
assert i != -1, "Impossible line %d to reflow: %s" % (size, `s`)
lines.append(padding + cur[:i])
if len(lines) == 1:
# find new size based on brace
j = cur.find('{', 0, i)
if j >= 0:
j += 2 # account for the brace and the space after it
size -= j
padding = " " * j
else:
j = cur.find('(', 0, i)
if j >= 0:
j += 1 # account for the paren (no space after it)
size -= j
padding = " " * j
cur = cur[i+1:]
else:
lines.append(padding + cur)
return lines
def is_simple(sum):
"""Return True if a sum is a simple.
A sum is simple if its types have no fields, e.g.
unaryop = Invert | Not | UAdd | USub
"""
for t in sum.types:
if t.fields:
return False
return True
class EmitVisitor(asdl.VisitorBase):
"""Visit that emits lines"""
def __init__(self, file):
self.file = file
super(EmitVisitor, self).__init__()
def emit(self, s, depth, reflow=1):
# XXX reflow long lines?
if reflow:
lines = reflow_lines(s, depth)
else:
lines = [s]
for line in lines:
line = (" " * TABSIZE * depth) + line + "\n"
self.file.write(line)
class TypeDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
if is_simple(sum):
self.simple_sum(sum, name, depth)
else:
self.sum_with_constructors(sum, name, depth)
def simple_sum(self, sum, name, depth):
enum = []
for i in range(len(sum.types)):
type = sum.types[i]
enum.append("%s=%d" % (type.name, i + 1))
enums = ", ".join(enum)
ctype = get_c_type(name)
s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
self.emit(s, depth)
self.emit("", depth)
def sum_with_constructors(self, sum, name, depth):
ctype = get_c_type(name)
s = "typedef struct _%(name)s *%(ctype)s;" % locals()
self.emit(s, depth)
self.emit("", depth)
def visitProduct(self, product, name, depth):
ctype = get_c_type(name)
s = "typedef struct _%(name)s *%(ctype)s;" % locals()
self.emit(s, depth)
self.emit("", depth)
class StructVisitor(EmitVisitor):
"""Visitor to generate typdefs for AST."""
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
if not is_simple(sum):
self.sum_with_constructors(sum, name, depth)
def sum_with_constructors(self, sum, name, depth):
def emit(s, depth=depth):
self.emit(s % sys._getframe(1).f_locals, depth)
enum = []
for i in range(len(sum.types)):
type = sum.types[i]
enum.append("%s_kind=%d" % (type.name, i + 1))
emit("struct _%(name)s {")
emit("enum { " + ", ".join(enum) + " } kind;", depth + 1)
emit("union {", depth + 1)
for t in sum.types:
self.visit(t, depth + 2)
emit("} v;", depth + 1)
for field in sum.attributes:
# rudimentary attribute handling
type = str(field.type)
assert type in asdl.builtin_types, type
emit("%s %s;" % (type, field.name), depth + 1);
emit("};")
emit("")
def visitConstructor(self, cons, depth):
if cons.fields:
self.emit("struct {", depth)
for f in cons.fields:
self.visit(f, depth + 1)
self.emit("} %s;" % cons.name, depth)
self.emit("", depth)
else:
# XXX not sure what I want here, nothing is probably fine
pass
def visitField(self, field, depth):
# XXX need to lookup field.type, because it might be something
# like a builtin...
ctype = get_c_type(field.type)
name = field.name
if field.seq:
self.emit("asdl_seq *%(name)s;" % locals(), depth)
else:
self.emit("%(ctype)s %(name)s;" % locals(), depth)
def visitProduct(self, product, name, depth):
self.emit("struct _%(name)s {" % locals(), depth)
for f in product.fields:
self.visit(f, depth + 1)
self.emit("};", depth)
self.emit("", depth)
class PrototypeVisitor(EmitVisitor):
"""Generate function prototypes for the .h file"""
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, type.name)
def visitSum(self, sum, name):
if is_simple(sum):
pass # XXX
else:
for t in sum.types:
self.visit(t, name, sum.attributes)
def get_args(self, fields):
"""Return list of C argument into, one for each field.
Argument info is 3-tuple of a C type, variable name, and flag
that is true if type can be NULL.
"""
args = []
unnamed = {}
for f in fields:
if f.name is None:
name = f.type
c = unnamed[name] = unnamed.get(name, 0) + 1
if c > 1:
name = "name%d" % (c - 1)
else:
name = f.name
# XXX should extend get_c_type() to handle this
if f.seq:
ctype = "asdl_seq *"
else:
ctype = get_c_type(f.type)
args.append((ctype, name, f.opt or f.seq))
return args
def visitConstructor(self, cons, type, attrs):
args = self.get_args(cons.fields)
attrs = self.get_args(attrs)
ctype = get_c_type(type)
self.emit_function(cons.name, ctype, args, attrs)
def emit_function(self, name, ctype, args, attrs, union=1):
args = args + attrs
if args:
argstr = ", ".join(["%s %s" % (atype, aname)
for atype, aname, opt in args])
else:
argstr = "void"
self.emit("%s %s(%s);" % (ctype, name, argstr), 0)
def visitProduct(self, prod, name):
self.emit_function(name, get_c_type(name),
self.get_args(prod.fields), [], union=0)
class FunctionVisitor(PrototypeVisitor):
"""Visitor to generate constructor functions for AST."""
def emit_function(self, name, ctype, args, attrs, union=1):
def emit(s, depth=0, reflow=1):
self.emit(s, depth, reflow)
argstr = ", ".join(["%s %s" % (atype, aname)
for atype, aname, opt in args + attrs])
self.emit("%s" % ctype, 0)
emit("%s(%s)" % (name, argstr))
emit("{")
emit("%s p;" % ctype, 1)
for argtype, argname, opt in args:
# XXX hack alert: false is allowed for a bool
if not opt and not argtype == "bool":
emit("if (!%s) {" % argname, 1)
emit("PyErr_SetString(PyExc_ValueError,", 2)
msg = "field %s is required for %s" % (argname, name)
emit(' "%s");' % msg,
2, reflow=0)
emit('return NULL;', 2)
emit('}', 1)
emit("p = (%s)malloc(sizeof(*p));" % ctype, 1)
emit("if (!p) {", 1)
emit("PyErr_SetString(PyExc_MemoryError, \"no memory\");", 2)
emit("return NULL;", 2)
emit("}", 1)
if union:
self.emit_body_union(name, args, attrs)
else:
self.emit_body_struct(name, args, attrs)
emit("return p;", 1)
emit("}")
emit("")
def emit_body_union(self, name, args, attrs):
def emit(s, depth=0, reflow=1):
self.emit(s, depth, reflow)
emit("p->kind = %s_kind;" % name, 1)
for argtype, argname, opt in args:
emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
for argtype, argname, opt in attrs:
emit("p->%s = %s;" % (argname, argname), 1)
def emit_body_struct(self, name, args, attrs):
def emit(s, depth=0, reflow=1):
self.emit(s, depth, reflow)
for argtype, argname, opt in args:
emit("p->%s = %s;" % (argname, argname), 1)
assert not attrs
class PickleVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, type.name)
def visitSum(self, sum, name):
pass
def visitProduct(self, sum, name):
pass
def visitConstructor(self, cons, name):
pass
def visitField(self, sum):
pass
class MarshalPrototypeVisitor(PickleVisitor):
def prototype(self, sum, name):
ctype = get_c_type(name)
self.emit("int marshal_write_%s(PyObject **, int *, %s);"
% (name, ctype), 0)
visitProduct = visitSum = prototype
class FreePrototypeVisitor(PickleVisitor):
def prototype(self, sum, name):
ctype = get_c_type(name)
self.emit("void free_%s(%s);" % (name, ctype), 0)
visitProduct = visitSum = prototype
_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
def find_sequence(fields, doing_specialization):
"""Return True if any field uses a sequence."""
for f in fields:
if f.seq:
if not doing_specialization:
return True
if str(f.type) not in _SPECIALIZED_SEQUENCES:
return True
return False
def has_sequence(types, doing_specialization):
for t in types:
if find_sequence(t.fields, doing_specialization):
return True
return False
class StaticVisitor(PickleVisitor):
'''Very simple, always emit this static code'''
CODE = '''static void
free_seq_exprs(asdl_seq *seq)
{
int i, n;
n = asdl_seq_LEN(seq);
for (i = 0; i < n; i++)
free_expr((expr_ty)asdl_seq_GET(seq, i));
asdl_seq_free(seq);
}
static void
free_seq_stmts(asdl_seq *seq)
{
int i, n;
n = asdl_seq_LEN(seq);
for (i = 0; i < n; i++)
free_stmt((stmt_ty)asdl_seq_GET(seq, i));
asdl_seq_free(seq);
}
'''
def visit(self, object):
self.emit(self.CODE, 0, reflow=False)
class FreeVisitor(PickleVisitor):
def func_begin(self, name, has_seq):
ctype = get_c_type(name)
self.emit("void", 0)
self.emit("free_%s(%s o)" % (name, ctype), 0)
self.emit("{", 0)
if has_seq:
self.emit("int i, n;", 1)
self.emit("asdl_seq *seq;", 1)
self.emit('', 0)
self.emit('if (!o)', 1)
self.emit('return;', 2)
self.emit('', 0)
def func_end(self):
self.emit("}", 0)
self.emit("", 0)
def visitSum(self, sum, name):
has_seq = has_sequence(sum.types, True)
self.func_begin(name, has_seq)
if not is_simple(sum):
self.emit("switch (o->kind) {", 1)
for i in range(len(sum.types)):
t = sum.types[i]
self.visitConstructor(t, i + 1, name)
self.emit("}", 1)
self.emit("", 0)
self.emit("free(o);", 1)
self.func_end()
def visitProduct(self, prod, name):
self.func_begin(name, find_sequence(prod.fields, True))
for field in prod.fields:
self.visitField(field, name, 1, True)
self.emit("", 0)
self.emit("free(o);", 1)
self.func_end()
def visitConstructor(self, cons, enum, name):
self.emit("case %s_kind:" % cons.name, 1)
for f in cons.fields:
self.visitField(f, cons.name, 2, False)
self.emit("break;", 2)
def visitField(self, field, name, depth, product):
def emit(s, d):
self.emit(s, depth + d)
if product:
value = "o->%s" % field.name
else:
value = "o->v.%s.%s" % (name, field.name)
if field.seq:
self.emitSeq(field, value, depth, emit)
# XXX need to know the simple types in advance, so that we
# don't call free_TYPE() for them.
elif field.opt:
emit("if (%s) {" % value, 0)
self.free(field, value, depth + 1)
emit("}", 0)
else:
self.free(field, value, depth)
def emitSeq(self, field, value, depth, emit):
# specialize for freeing sequences of statements and expressions
if str(field.type) in _SPECIALIZED_SEQUENCES:
c_code = "free_seq_%ss(%s);" % (field.type, value)
emit(c_code, 0)
else:
emit("seq = %s;" % value, 0)
emit("n = asdl_seq_LEN(seq);", 0)
emit("for (i = 0; i < n; i++)", 0)
self.free(field, "asdl_seq_GET(seq, i)", depth + 1)
emit("asdl_seq_free(seq);", 0)
def free(self, field, value, depth):
if str(field.type) in ("identifier", "string", "object"):
ctype = get_c_type(field.type)
self.emit("Py_DECREF((%s)%s);" % (ctype, value), depth)
elif str(field.type) == "bool":
return
else:
ctype = get_c_type(field.type)
self.emit("free_%s((%s)%s);" % (field.type, ctype, value), depth)
class MarshalFunctionVisitor(PickleVisitor):
def func_begin(self, name, has_seq):
ctype = get_c_type(name)
self.emit("int", 0)
self.emit("marshal_write_%s(PyObject **buf, int *off, %s o)" %
(name, ctype), 0)
self.emit("{", 0)
# XXX: add declaration of "int i;" properly
if has_seq or True:
self.emit("int i;", 1) # XXX only need it for sequences
def func_end(self):
self.emit("return 1;", 1)
self.emit("}", 0)
self.emit("", 0)
def visitSum(self, sum, name):
has_seq = has_sequence(sum.types, False)
self.func_begin(name, has_seq)
simple = is_simple(sum)
if simple:
self.emit("switch (o) {", 1)
else:
self.emit("switch (o->kind) {", 1)
for i in range(len(sum.types)):
t = sum.types[i]
self.visitConstructor(t, i + 1, name, simple)
self.emit("}", 1)
self.func_end()
def visitProduct(self, prod, name):
self.func_begin(name, find_sequence(prod.fields, True))
for field in prod.fields:
self.visitField(field, name, 1, 1)
self.func_end()
def visitConstructor(self, cons, enum, name, simple):
if simple:
self.emit("case %s:" % cons.name, 1)
self.emit("marshal_write_int(buf, off, %d);" % enum, 2);
self.emit("break;", 2)
else:
self.emit("case %s_kind:" % cons.name, 1)
self.emit("marshal_write_int(buf, off, %d);" % enum, 2)
for f in cons.fields:
self.visitField(f, cons.name, 2, 0)
self.emit("break;", 2)
def visitField(self, field, name, depth, product):
def emit(s, d):
self.emit(s, depth + d)
if product:
value = "o->%s" % field.name
else:
value = "o->v.%s.%s" % (name, field.name)
if field.seq:
emit("marshal_write_int(buf, off, asdl_seq_LEN(%s));" % value, 0)
emit("for (i = 0; i < asdl_seq_LEN(%s); i++) {" % value, 0)
emit("void *elt = asdl_seq_GET(%s, i);" % value, 1);
ctype = get_c_type(field.type);
emit("marshal_write_%s(buf, off, (%s)elt);" % (field.type,
ctype), 1)
emit("}", 0)
elif field.opt:
emit("if (%s) {" % value, 0)
emit("marshal_write_int(buf, off, 1);", 1)
emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 1)
emit("}", 0)
emit("else {", 0)
emit("marshal_write_int(buf, off, 0);", 1)
emit("}", 0)
else:
emit("marshal_write_%s(buf, off, %s);" % (field.type, value), 0)
class ChainOfVisitors:
def __init__(self, *visitors):
self.visitors = visitors
def visit(self, object):
for v in self.visitors:
v.visit(object)
def main(srcfile):
auto_gen_msg = '/* File automatically generated by %s */\n' % sys.argv[0]
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
else:
p = "%s-ast.h" % mod.name
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "asdl.h"\n'
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
FreePrototypeVisitor(f),
MarshalPrototypeVisitor(f),
)
c.visit(mod)
f.close()
if SRC_DIR:
p = "%s/%s-ast.c" % (SRC_DIR, mod.name)
else:
p = "%s-ast.c" % mod.name
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "Python.h"'
print >> f, '#include "%s-ast.h"' % mod.name
print >> f
v = ChainOfVisitors(FunctionVisitor(f),
StaticVisitor(f),
FreeVisitor(f),
MarshalFunctionVisitor(f),
)
v.visit(mod)
f.close()
if __name__ == "__main__":
import sys
import getopt
INC_DIR = ''
SRC_DIR = ''
opts, args = getopt.getopt(sys.argv[1:], "h:c:")
for o, v in opts:
if o == '-h':
INC_DIR = v
if o == '-c':
SRC_DIR = v
if len(args) != 1:
print "Must specify single input file"
main(args[0])

2
Parser/grammar.mak
View File

@ -15,7 +15,7 @@
# particular case --pragma in PC\pyconfig.h, which demands that
# python23.lib get linked in).
LIBS= ..\PCbuild\python23.lib
LIBS= ..\PCbuild\python25.lib
CFLAGS= /I ..\Include /I ..\PC /D MS_NO_COREDLL /D PGEN /MD

5
Parser/parsetok.c
View File

@ -21,7 +21,7 @@ static void initerr(perrdetail *err_ret, const char* filename);
node *
PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
{
return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
}
node *
@ -56,7 +56,6 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
return parsetok(tok, g, start, err_ret, flags);
}
/* Parse input coming from a file. Return error code, print some errors. */
node *
@ -210,7 +209,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
}
static void
initerr(perrdetail *err_ret, const char* filename)
initerr(perrdetail *err_ret, const char *filename)
{
err_ret->error = E_OK;
err_ret->filename = filename;

840
Parser/spark.py Normal file
View File

@ -0,0 +1,840 @@
# Copyright (c) 1998-2002 John Aycock
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
__version__ = 'SPARK-0.7 (pre-alpha-5)'
import re
import sys
import string
def _namelist(instance):
namelist, namedict, classlist = [], {}, [instance.__class__]
for c in classlist:
for b in c.__bases__:
classlist.append(b)
for name in c.__dict__.keys():
if not namedict.has_key(name):
namelist.append(name)
namedict[name] = 1
return namelist
class GenericScanner:
def __init__(self, flags=0):
pattern = self.reflect()
self.re = re.compile(pattern, re.VERBOSE|flags)
self.index2func = {}
for name, number in self.re.groupindex.items():
self.index2func[number-1] = getattr(self, 't_' + name)
def makeRE(self, name):
doc = getattr(self, name).__doc__
rv = '(?P<%s>%s)' % (name[2:], doc)
return rv
def reflect(self):
rv = []
for name in _namelist(self):
if name[:2] == 't_' and name != 't_default':
rv.append(self.makeRE(name))
rv.append(self.makeRE('t_default'))
return string.join(rv, '|')
def error(self, s, pos):
print "Lexical error at position %s" % pos
raise SystemExit
def tokenize(self, s):
pos = 0
n = len(s)
while pos < n:
m = self.re.match(s, pos)
if m is None:
self.error(s, pos)
groups = m.groups()
for i in range(len(groups)):
if groups[i] and self.index2func.has_key(i):
self.index2func[i](groups[i])
pos = m.end()
def t_default(self, s):
r'( . | \n )+'
print "Specification error: unmatched input"
raise SystemExit
#
# Extracted from GenericParser and made global so that [un]picking works.
#
class _State:
def __init__(self, stateno, items):
self.T, self.complete, self.items = [], [], items
self.stateno = stateno
class GenericParser:
#
# An Earley parser, as per J. Earley, "An Efficient Context-Free
# Parsing Algorithm", CACM 13(2), pp. 94-102. Also J. C. Earley,
# "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
# Carnegie-Mellon University, August 1968. New formulation of
# the parser according to J. Aycock, "Practical Earley Parsing
# and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
# 2001, and J. Aycock and R. N. Horspool, "Practical Earley
# Parsing", unpublished paper, 2001.
#
def __init__(self, start):
self.rules = {}
self.rule2func = {}
self.rule2name = {}
self.collectRules()
self.augment(start)
self.ruleschanged = 1
_NULLABLE = '\e_'
_START = 'START'
_BOF = '|-'
#
# When pickling, take the time to generate the full state machine;
# some information is then extraneous, too. Unfortunately we
# can't save the rule2func map.
#
def __getstate__(self):
if self.ruleschanged:
#
# XXX - duplicated from parse()
#
self.computeNull()
self.newrules = {}
self.new2old = {}
self.makeNewRules()
self.ruleschanged = 0
self.edges, self.cores = {}, {}
self.states = { 0: self.makeState0() }
self.makeState(0, self._BOF)
#
# XXX - should find a better way to do this..
#
changes = 1
while changes:
changes = 0
for k, v in self.edges.items():
if v is None:
state, sym = k
if self.states.has_key(state):
self.goto(state, sym)
changes = 1
rv = self.__dict__.copy()
for s in self.states.values():
del s.items
del rv['rule2func']
del rv['nullable']
del rv['cores']
return rv
def __setstate__(self, D):
self.rules = {}
self.rule2func = {}
self.rule2name = {}
self.collectRules()
start = D['rules'][self._START][0][1][1] # Blech.
self.augment(start)
D['rule2func'] = self.rule2func
D['makeSet'] = self.makeSet_fast
self.__dict__ = D
#
# A hook for GenericASTBuilder and GenericASTMatcher. Mess
# thee not with this; nor shall thee toucheth the _preprocess
# argument to addRule.
#
def preprocess(self, rule, func): return rule, func
def addRule(self, doc, func, _preprocess=1):
fn = func
rules = string.split(doc)
index = []
for i in range(len(rules)):
if rules[i] == '::=':
index.append(i-1)
index.append(len(rules))
for i in range(len(index)-1):
lhs = rules[index[i]]
rhs = rules[index[i]+2:index[i+1]]
rule = (lhs, tuple(rhs))
if _preprocess:
rule, fn = self.preprocess(rule, func)
if self.rules.has_key(lhs):
self.rules[lhs].append(rule)
else:
self.rules[lhs] = [ rule ]
self.rule2func[rule] = fn
self.rule2name[rule] = func.__name__[2:]
self.ruleschanged = 1
def collectRules(self):
for name in _namelist(self):
if name[:2] == 'p_':
func = getattr(self, name)
doc = func.__doc__
self.addRule(doc, func)
def augment(self, start):
rule = '%s ::= %s %s' % (self._START, self._BOF, start)
self.addRule(rule, lambda args: args[1], 0)
def computeNull(self):
self.nullable = {}
tbd = []
for rulelist in self.rules.values():
lhs = rulelist[0][0]
self.nullable[lhs] = 0
for rule in rulelist:
rhs = rule[1]
if len(rhs) == 0:
self.nullable[lhs] = 1
continue
#
# We only need to consider rules which
# consist entirely of nonterminal symbols.
# This should be a savings on typical
# grammars.
#
for sym in rhs:
if not self.rules.has_key(sym):
break
else:
tbd.append(rule)
changes = 1
while changes:
changes = 0
for lhs, rhs in tbd:
if self.nullable[lhs]:
continue
for sym in rhs:
if not self.nullable[sym]:
break
else:
self.nullable[lhs] = 1
changes = 1
def makeState0(self):
s0 = _State(0, [])
for rule in self.newrules[self._START]:
s0.items.append((rule, 0))
return s0
def finalState(self, tokens):
#
# Yuck.
#
if len(self.newrules[self._START]) == 2 and len(tokens) == 0:
return 1
start = self.rules[self._START][0][1][1]
return self.goto(1, start)
def makeNewRules(self):
worklist = []
for rulelist in self.rules.values():
for rule in rulelist:
worklist.append((rule, 0, 1, rule))
for rule, i, candidate, oldrule in worklist:
lhs, rhs = rule
n = len(rhs)
while i < n:
sym = rhs[i]
if not self.rules.has_key(sym) or \
not self.nullable[sym]:
candidate = 0
i = i + 1
continue
newrhs = list(rhs)
newrhs[i] = self._NULLABLE+sym
newrule = (lhs, tuple(newrhs))
worklist.append((newrule, i+1,
candidate, oldrule))
candidate = 0
i = i + 1
else:
if candidate:
lhs = self._NULLABLE+lhs
rule = (lhs, rhs)
if self.newrules.has_key(lhs):
self.newrules[lhs].append(rule)
else:
self.newrules[lhs] = [ rule ]
self.new2old[rule] = oldrule
def typestring(self, token):
return None
def error(self, token):
print "Syntax error at or near `%s' token" % token
raise SystemExit
def parse(self, tokens):
sets = [ [(1,0), (2,0)] ]
self.links = {}
if self.ruleschanged:
self.computeNull()
self.newrules = {}
self.new2old = {}
self.makeNewRules()
self.ruleschanged = 0
self.edges, self.cores = {}, {}
self.states = { 0: self.makeState0() }
self.makeState(0, self._BOF)
for i in xrange(len(tokens)):
sets.append([])
if sets[i] == []:
break
self.makeSet(tokens[i], sets, i)
else:
sets.append([])
self.makeSet(None, sets, len(tokens))
#_dump(tokens, sets, self.states)
finalitem = (self.finalState(tokens), 0)
if finalitem not in sets[-2]:
if len(tokens) > 0:
self.error(tokens[i-1])
else:
self.error(None)
return self.buildTree(self._START, finalitem,
tokens, len(sets)-2)
def isnullable(self, sym):
#
# For symbols in G_e only. If we weren't supporting 1.5,
# could just use sym.startswith().
#
return self._NULLABLE == sym[0:len(self._NULLABLE)]
def skip(self, (lhs, rhs), pos=0):
n = len(rhs)
while pos < n:
if not self.isnullable(rhs[pos]):
break
pos = pos + 1
return pos
def makeState(self, state, sym):
assert sym is not None
#
# Compute \epsilon-kernel state's core and see if
# it exists already.
#
kitems = []
for rule, pos in self.states[state].items:
lhs, rhs = rule
if rhs[pos:pos+1] == (sym,):
kitems.append((rule, self.skip(rule, pos+1)))
core = kitems
core.sort()
tcore = tuple(core)
if self.cores.has_key(tcore):
return self.cores[tcore]
#
# Nope, doesn't exist. Compute it and the associated
# \epsilon-nonkernel state together; we'll need it right away.
#
k = self.cores[tcore] = len(self.states)
K, NK = _State(k, kitems), _State(k+1, [])
self.states[k] = K
predicted = {}
edges = self.edges
rules = self.newrules
for X in K, NK:
worklist = X.items
for item in worklist:
rule, pos = item
lhs, rhs = rule
if pos == len(rhs):
X.complete.append(rule)
continue
nextSym = rhs[pos]
key = (X.stateno, nextSym)
if not rules.has_key(nextSym):
if not edges.has_key(key):
edges[key] = None
X.T.append(nextSym)
else:
edges[key] = None
if not predicted.has_key(nextSym):
predicted[nextSym] = 1
for prule in rules[nextSym]:
ppos = self.skip(prule)
new = (prule, ppos)
NK.items.append(new)
#
# Problem: we know K needs generating, but we
# don't yet know about NK. Can't commit anything
# regarding NK to self.edges until we're sure. Should
# we delay committing on both K and NK to avoid this
# hacky code? This creates other problems..
#
if X is K:
edges = {}
if NK.items == []:
return k
#
# Check for \epsilon-nonkernel's core. Unfortunately we
# need to know the entire set of predicted nonterminals
# to do this without accidentally duplicating states.
#
core = predicted.keys()
core.sort()
tcore = tuple(core)
if self.cores.has_key(tcore):
self.edges[(k, None)] = self.cores[tcore]
return k
nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno
self.edges.update(edges)
self.states[nk] = NK
return k
def goto(self, state, sym):
key = (state, sym)
if not self.edges.has_key(key):
#
# No transitions from state on sym.
#
return None
rv = self.edges[key]
if rv is None:
#
# Target state isn't generated yet. Remedy this.
#
rv = self.makeState(state, sym)
self.edges[key] = rv
return rv
def gotoT(self, state, t):
return [self.goto(state, t)]
def gotoST(self, state, st):
rv = []
for t in self.states[state].T:
if st == t:
rv.append(self.goto(state, t))
return rv
def add(self, set, item, i=None, predecessor=None, causal=None):
if predecessor is None:
if item not in set:
set.append(item)
else:
key = (item, i)
if item not in set:
self.links[key] = []
set.append(item)
self.links[key].append((predecessor, causal))
def makeSet(self, token, sets, i):
cur, next = sets[i], sets[i+1]
ttype = token is not None and self.typestring(token) or None
if ttype is not None:
fn, arg = self.gotoT, ttype
else:
fn, arg = self.gotoST, token
for item in cur:
ptr = (item, i)
state, parent = item
add = fn(state, arg)
for k in add:
if k is not None:
self.add(next, (k, parent), i+1, ptr)
nk = self.goto(k, None)
if nk is not None:
self.add(next, (nk, i+1))
if parent == i:
continue
for rule in self.states[state].complete:
lhs, rhs = rule
for pitem in sets[parent]:
pstate, pparent = pitem
k = self.goto(pstate, lhs)
if k is not None:
why = (item, i, rule)
pptr = (pitem, parent)
self.add(cur, (k, pparent),
i, pptr, why)
nk = self.goto(k, None)
if nk is not None:
self.add(cur, (nk, i))
def makeSet_fast(self, token, sets, i):
#
# Call *only* when the entire state machine has been built!
# It relies on self.edges being filled in completely, and
# then duplicates and inlines code to boost speed at the
# cost of extreme ugliness.
#
cur, next = sets[i], sets[i+1]
ttype = token is not None and self.typestring(token) or None
for item in cur:
ptr = (item, i)
state, parent = item
if ttype is not None:
k = self.edges.get((state, ttype), None)
if k is not None:
#self.add(next, (k, parent), i+1, ptr)
#INLINED --v
new = (k, parent)
key = (new, i+1)
if new not in next:
self.links[key] = []
next.append(new)
self.links[key].append((ptr, None))
#INLINED --^
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
#self.add(next, (nk, i+1))
#INLINED --v
new = (nk, i+1)
if new not in next:
next.append(new)
#INLINED --^
else:
add = self.gotoST(state, token)
for k in add:
if k is not None:
self.add(next, (k, parent), i+1, ptr)
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
self.add(next, (nk, i+1))
if parent == i:
continue
for rule in self.states[state].complete:
lhs, rhs = rule
for pitem in sets[parent]:
pstate, pparent = pitem
#k = self.goto(pstate, lhs)
k = self.edges.get((pstate, lhs), None)
if k is not None:
why = (item, i, rule)
pptr = (pitem, parent)
#self.add(cur, (k, pparent),
# i, pptr, why)
#INLINED --v
new = (k, pparent)
key = (new, i)
if new not in cur:
self.links[key] = []
cur.append(new)
self.links[key].append((pptr, why))
#INLINED --^
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
#self.add(cur, (nk, i))
#INLINED --v
new = (nk, i)
if new not in cur:
cur.append(new)
#INLINED --^
def predecessor(self, key, causal):
for p, c in self.links[key]:
if c == causal:
return p
assert 0
def causal(self, key):
links = self.links[key]
if len(links) == 1:
return links[0][1]
choices = []
rule2cause = {}
for p, c in links:
rule = c[2]
choices.append(rule)
rule2cause[rule] = c
return rule2cause[self.ambiguity(choices)]
def deriveEpsilon(self, nt):
if len(self.newrules[nt]) > 1:
rule = self.ambiguity(self.newrules[nt])
else:
rule = self.newrules[nt][0]
#print rule
rhs = rule[1]
attr = [None] * len(rhs)
for i in range(len(rhs)-1, -1, -1):
attr[i] = self.deriveEpsilon(rhs[i])
return self.rule2func[self.new2old[rule]](attr)
def buildTree(self, nt, item, tokens, k):
state, parent = item
choices = []
for rule in self.states[state].complete:
if rule[0] == nt:
choices.append(rule)
rule = choices[0]
if len(choices) > 1:
rule = self.ambiguity(choices)
#print rule
rhs = rule[1]
attr = [None] * len(rhs)
for i in range(len(rhs)-1, -1, -1):
sym = rhs[i]
if not self.newrules.has_key(sym):
if sym != self._BOF:
attr[i] = tokens[k-1]
key = (item, k)
item, k = self.predecessor(key, None)
#elif self.isnullable(sym):
elif self._NULLABLE == sym[0:len(self._NULLABLE)]:
attr[i] = self.deriveEpsilon(sym)
else:
key = (item, k)
why = self.causal(key)
attr[i] = self.buildTree(sym, why[0],
tokens, why[1])
item, k = self.predecessor(key, why)
return self.rule2func[self.new2old[rule]](attr)
def ambiguity(self, rules):
#
# XXX - problem here and in collectRules() if the same rule
# appears in >1 method. Also undefined results if rules
# causing the ambiguity appear in the same method.
#
sortlist = []
name2index = {}
for i in range(len(rules)):
lhs, rhs = rule = rules[i]
name = self.rule2name[self.new2old[rule]]
sortlist.append((len(rhs), name))
name2index[name] = i
sortlist.sort()
list = map(lambda (a,b): b, sortlist)
return rules[name2index[self.resolve(list)]]
def resolve(self, list):
#
# Resolve ambiguity in favor of the shortest RHS.
# Since we walk the tree from the top down, this
# should effectively resolve in favor of a "shift".
#
return list[0]
#
# GenericASTBuilder automagically constructs a concrete/abstract syntax tree
# for a given input. The extra argument is a class (not an instance!)
# which supports the "__setslice__" and "__len__" methods.
#
# XXX - silently overrides any user code in methods.
#
class GenericASTBuilder(GenericParser):
def __init__(self, AST, start):
GenericParser.__init__(self, start)
self.AST = AST
def preprocess(self, rule, func):
rebind = lambda lhs, self=self: \
lambda args, lhs=lhs, self=self: \
self.buildASTNode(args, lhs)
lhs, rhs = rule
return rule, rebind(lhs)
def buildASTNode(self, args, lhs):
children = []
for arg in args:
if isinstance(arg, self.AST):
children.append(arg)
else:
children.append(self.terminal(arg))
return self.nonterminal(lhs, children)
def terminal(self, token): return token
def nonterminal(self, type, args):
rv = self.AST(type)
rv[:len(args)] = args
return rv
#
# GenericASTTraversal is a Visitor pattern according to Design Patterns. For
# each node it attempts to invoke the method n_<node type>, falling
# back onto the default() method if the n_* can't be found. The preorder
# traversal also looks for an exit hook named n_<node type>_exit (no default
# routine is called if it's not found). To prematurely halt traversal
# of a subtree, call the prune() method -- this only makes sense for a
# preorder traversal. Node type is determined via the typestring() method.
#
class GenericASTTraversalPruningException:
pass
class GenericASTTraversal:
def __init__(self, ast):
self.ast = ast
def typestring(self, node):
return node.type
def prune(self):
raise GenericASTTraversalPruningException
def preorder(self, node=None):
if node is None:
node = self.ast
try:
name = 'n_' + self.typestring(node)
if hasattr(self, name):
func = getattr(self, name)
func(node)
else:
self.default(node)
except GenericASTTraversalPruningException:
return
for kid in node:
self.preorder(kid)
name = name + '_exit'
if hasattr(self, name):
func = getattr(self, name)
func(node)
def postorder(self, node=None):
if node is None:
node = self.ast
for kid in node:
self.postorder(kid)
name = 'n_' + self.typestring(node)
if hasattr(self, name):
func = getattr(self, name)
func(node)
else:
self.default(node)
def default(self, node):
pass
#
# GenericASTMatcher. AST nodes must have "__getitem__" and "__cmp__"
# implemented.
#
# XXX - makes assumptions about how GenericParser walks the parse tree.
#
class GenericASTMatcher(GenericParser):
def __init__(self, start, ast):
GenericParser.__init__(self, start)
self.ast = ast
def preprocess(self, rule, func):
rebind = lambda func, self=self: \
lambda args, func=func, self=self: \
self.foundMatch(args, func)
lhs, rhs = rule
rhslist = list(rhs)
rhslist.reverse()
return (lhs, tuple(rhslist)), rebind(func)
def foundMatch(self, args, func):
func(args[-1])
return args[-1]
def match_r(self, node):
self.input.insert(0, node)
children = 0
for child in node:
if children == 0:
self.input.insert(0, '(')
children = children + 1
self.match_r(child)
if children > 0:
self.input.insert(0, ')')
def match(self, ast=None):
if ast is None:
ast = self.ast
self.input = []
self.match_r(ast)
self.parse(self.input)
def resolve(self, list):
#
# Resolve ambiguity in favor of the longest RHS.
#
return list[-1]
def _dump(tokens, sets, states):
for i in range(len(sets)):
print 'set', i
for item in sets[i]:
print '\t', item
for (lhs, rhs), pos in states[item[0]].items:
print '\t\t', lhs, '::=',
print string.join(rhs[:pos]),
print '.',
print string.join(rhs[pos:])
if i < len(tokens):
print
print 'token', str(tokens[i])
print

2281
Python/Python-ast.c Normal file
View File

File diff suppressed because it is too large Load Diff

92
Python/asdl.c Normal file
View File

@ -0,0 +1,92 @@
#include "Python.h"
#include "asdl.h"
asdl_seq *
asdl_seq_new(int size)
{
asdl_seq *seq = NULL;
size_t n = sizeof(asdl_seq) +
(size ? (sizeof(void *) * (size - 1)) : 0);
seq = (asdl_seq *)PyObject_Malloc(n);
if (!seq) {
PyErr_SetString(PyExc_MemoryError, "no memory");
return NULL;
}
memset(seq, 0, n);
seq->size = size;
return seq;
}
void
asdl_seq_free(asdl_seq *seq)
{
PyObject_Free(seq);
}
#define CHECKSIZE(BUF, OFF, MIN) { \
int need = *(OFF) + MIN; \
if (need >= PyString_GET_SIZE(*(BUF))) { \
int newsize = PyString_GET_SIZE(*(BUF)) * 2; \
if (newsize < need) \
newsize = need; \
if (_PyString_Resize((BUF), newsize) < 0) \
return 0; \
} \
}
int
marshal_write_int(PyObject **buf, int *offset, int x)
{
char *s;
CHECKSIZE(buf, offset, 4)
s = PyString_AS_STRING(*buf) + (*offset);
s[0] = (x & 0xff);
s[1] = (x >> 8) & 0xff;
s[2] = (x >> 16) & 0xff;
s[3] = (x >> 24) & 0xff;
*offset += 4;
return 1;
}
int
marshal_write_bool(PyObject **buf, int *offset, bool b)
{
if (b)
marshal_write_int(buf, offset, 1);
else
marshal_write_int(buf, offset, 0);
return 1;
}
int
marshal_write_identifier(PyObject **buf, int *offset, identifier id)
{
int l = PyString_GET_SIZE(id);
marshal_write_int(buf, offset, l);
CHECKSIZE(buf, offset, l);
memcpy(PyString_AS_STRING(*buf) + *offset,
PyString_AS_STRING(id), l);
*offset += l;
return 1;
}
int
marshal_write_string(PyObject **buf, int *offset, string s)
{
int len = PyString_GET_SIZE(s);
marshal_write_int(buf, offset, len);
CHECKSIZE(buf, offset, len);
memcpy(PyString_AS_STRING(*buf) + *offset,
PyString_AS_STRING(s), len);
*offset += len;
return 1;
}
int
marshal_write_object(PyObject **buf, int *offset, object s)
{
/* XXX */
return 0;
}

3114
Python/ast.c Normal file
View File

File diff suppressed because it is too large Load Diff

2
Python/bltinmodule.c
View File

@ -1,9 +1,9 @@
/* Built-in functions */
#include "Python.h"
#include "node.h"
#include "code.h"
#include "compile.h"
#include "eval.h"

45
Python/ceval.c
View File

@ -8,7 +8,7 @@
#include "Python.h"
#include "compile.h"
#include "code.h"
#include "frameobject.h"
#include "eval.h"
#include "opcode.h"
@ -543,7 +543,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
#ifdef LLTRACE
int lltrace;
#endif
#if defined(Py_DEBUG) || defined(LLTRACE)
#if defined(Py_DEBUG)
/* Make it easier to find out where we are with a debugger */
char *filename;
#endif
@ -743,9 +743,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
#ifdef LLTRACE
lltrace = PyDict_GetItemString(f->f_globals,"__lltrace__") != NULL;
lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
#endif
#if defined(Py_DEBUG) || defined(LLTRACE)
#if defined(Py_DEBUG)
filename = PyString_AsString(co->co_filename);
#endif
@ -2257,23 +2257,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throw)
case MAKE_CLOSURE:
{
int nfree;
v = POP(); /* code object */
x = PyFunction_New(v, f->f_globals);
nfree = PyCode_GetNumFree((PyCodeObject *)v);
Py_DECREF(v);
/* XXX Maybe this should be a separate opcode? */
if (x != NULL && nfree > 0) {
v = PyTuple_New(nfree);
if (v == NULL) {
Py_DECREF(x);
x = NULL;
break;
}
while (--nfree >= 0) {
w = POP();
PyTuple_SET_ITEM(v, nfree, w);
}
if (x != NULL) {
v = POP();
err = PyFunction_SetClosure(x, v);
Py_DECREF(v);
}
@ -2695,12 +2683,18 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
if (co->co_flags & CO_VARKEYWORDS)
nargs++;
/* Check for cells that shadow args */
for (i = 0; i < f->f_ncells && j < nargs; ++i) {
/* Initialize each cell var, taking into account
cell vars that are initialized from arguments.
Should arrange for the compiler to put cellvars
that are arguments at the beginning of the cellvars
list so that we can march over it more efficiently?
*/
for (i = 0; i < f->f_ncells; ++i) {
cellname = PyString_AS_STRING(
PyTuple_GET_ITEM(co->co_cellvars, i));
found = 0;
while (j < nargs) {
for (j = 0; j < nargs; j++) {
argname = PyString_AS_STRING(
PyTuple_GET_ITEM(co->co_varnames, j));
if (strcmp(cellname, argname) == 0) {
@ -2711,7 +2705,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
found = 1;
break;
}
j++;
}
if (found == 0) {
c = PyCell_New(NULL);
@ -2720,14 +2713,6 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
SETLOCAL(f->f_nlocals + i, c);
}
}
/* Initialize any that are left */
while (i < f->f_ncells) {
c = PyCell_New(NULL);
if (c == NULL)
goto fail;
SETLOCAL(f->f_nlocals + i, c);
i++;
}
}
if (f->f_nfreevars) {
int i;

9391
Python/compile.c
View File

File diff suppressed because it is too large Load Diff

282
Python/future.c
View File

@ -1,37 +1,30 @@
#include "Python.h"
#include "Python-ast.h"
#include "node.h"
#include "token.h"
#include "graminit.h"
#include "code.h"
#include "compile.h"
#include "symtable.h"
#define UNDEFINED_FUTURE_FEATURE "future feature %.100s is not defined"
#define FUTURE_IMPORT_STAR "future statement does not support import *"
/* FUTURE_POSSIBLE() is provided to accomodate doc strings, which is
the only statement that can occur before a future statement.
*/
#define FUTURE_POSSIBLE(FF) ((FF)->ff_last_lineno == -1)
static int
future_check_features(PyFutureFeatures *ff, node *n, const char *filename)
future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
{
int i;
char *feature;
node *ch, *nn;
const char *feature;
asdl_seq *names;
REQ(n, import_from);
nn = CHILD(n, 3 + (TYPE(CHILD(n, 3)) == LPAR));
if (TYPE(nn) == STAR) {
PyErr_SetString(PyExc_SyntaxError, FUTURE_IMPORT_STAR);
PyErr_SyntaxLocation(filename, nn->n_lineno);
return -1;
}
REQ(nn, import_as_names);
for (i = 0; i < NCH(nn); i += 2) {
ch = CHILD(nn, i);
REQ(ch, import_as_name);
feature = STR(CHILD(ch, 0));
assert(s->kind == ImportFrom_kind);
names = s->v.ImportFrom.names;
for (i = 0; i < asdl_seq_LEN(names); i++) {
alias_ty name = asdl_seq_GET(names, i);
feature = PyString_AsString(name->name);
if (!feature)
return 0;
if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
continue;
} else if (strcmp(feature, FUTURE_GENERATORS) == 0) {
@ -41,218 +34,97 @@ future_check_features(PyFutureFeatures *ff, node *n, const char *filename)
} else if (strcmp(feature, "braces") == 0) {
PyErr_SetString(PyExc_SyntaxError,
"not a chance");
PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno);
return -1;
PyErr_SyntaxLocation(filename, s->lineno);
return 0;
} else {
PyErr_Format(PyExc_SyntaxError,
UNDEFINED_FUTURE_FEATURE, feature);
PyErr_SyntaxLocation(filename, CHILD(ch, 0)->n_lineno);
return -1;
PyErr_SyntaxLocation(filename, s->lineno);
return 0;
}
}
return 0;
return 1;
}
static void
future_error(node *n, const char *filename)
int
future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename)
{
PyErr_SetString(PyExc_SyntaxError,
"from __future__ imports must occur at the "
"beginning of the file");
PyErr_SyntaxLocation(filename, n->n_lineno);
}
int i, found_docstring = 0, done = 0, prev_line = 0;
/* Relevant portions of the grammar:
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
file_input: (NEWLINE | stmt)* ENDMARKER
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt
| import_stmt | global_stmt | exec_stmt | assert_stmt
import_stmt: 'import' dotted_as_name (',' dotted_as_name)*
| 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
import_as_name: NAME [NAME NAME]
dotted_as_name: dotted_name [NAME NAME]
dotted_name: NAME ('.' NAME)*
*/
/* future_parse() finds future statements at the beginnning of a
module. The function calls itself recursively, rather than
factoring out logic for different kinds of statements into
different routines.
Return values:
-1 indicates an error occurred, e.g. unknown feature name
0 indicates no feature was found
1 indicates a feature was found
*/
static int
future_parse(PyFutureFeatures *ff, node *n, const char *filename)
{
int i, r;
loop:
switch (TYPE(n)) {
case single_input:
if (TYPE(CHILD(n, 0)) == simple_stmt) {
n = CHILD(n, 0);
goto loop;
}
return 0;
case file_input:
/* Check each statement in the file, starting with the
first, and continuing until the first statement
that isn't a future statement.
*/
for (i = 0; i < NCH(n); i++) {
node *ch = CHILD(n, i);
if (TYPE(ch) == stmt) {
r = future_parse(ff, ch, filename);
/* Need to check both conditions below
to accomodate doc strings, which
causes r < 0.
*/
if (r < 1 && !FUTURE_POSSIBLE(ff))
return r;
}
}
return 0;
case simple_stmt:
if (NCH(n) == 2) {
REQ(CHILD(n, 0), small_stmt);
n = CHILD(n, 0);
goto loop;
} else {
/* Deal with the special case of a series of
small statements on a single line. If a
future statement follows some other
statement, the SyntaxError is raised here.
In all other cases, the symtable pass
raises the exception.
*/
int found = 0, end_of_future = 0;
for (i = 0; i < NCH(n); i += 2) {
if (TYPE(CHILD(n, i)) == small_stmt) {
r = future_parse(ff, CHILD(n, i),
filename);
if (r < 1)
end_of_future = 1;
else {
found = 1;
if (end_of_future) {
future_error(n,
filename);
return -1;
}
}
}
}
/* If we found one and only one, then the
current lineno is legal.
*/
if (found)
ff->ff_last_lineno = n->n_lineno + 1;
else
ff->ff_last_lineno = n->n_lineno;
if (end_of_future && found)
return 1;
else
return 0;
}
case stmt:
if (TYPE(CHILD(n, 0)) == simple_stmt) {
n = CHILD(n, 0);
goto loop;
} else if (TYPE(CHILD(n, 0)) == expr_stmt) {
n = CHILD(n, 0);
goto loop;
} else {
REQ(CHILD(n, 0), compound_stmt);
ff->ff_last_lineno = n->n_lineno;
static PyObject *future;
if (!future) {
future = PyString_InternFromString("__future__");
if (!future)
return 0;
}
}
case small_stmt:
n = CHILD(n, 0);
goto loop;
case import_stmt: {
node *name;
n = CHILD(n, 0);
if (TYPE(n) != import_from) {
ff->ff_last_lineno = n->n_lineno;
return 0;
}
name = CHILD(n, 1);
if (strcmp(STR(CHILD(name, 0)), "__future__") != 0)
return 0;
if (future_check_features(ff, n, filename) < 0)
return -1;
ff->ff_last_lineno = n->n_lineno + 1;
if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
return 1;
}
/* The cases below -- all of them! -- are necessary to find
and skip doc strings. */
case expr_stmt:
case testlist:
case test:
case and_test:
case not_test:
case comparison:
case expr:
case xor_expr:
case and_expr:
case shift_expr:
case arith_expr:
case term:
case factor:
case power:
if (NCH(n) == 1) {
n = CHILD(n, 0);
goto loop;
/* A subsequent pass will detect future imports that don't
appear at the beginning of the file. There's one case,
however, that is easier to handl here: A series of imports
joined by semi-colons, where the first import is a future
statement but some subsequent import has the future form
but is preceded by a regular import.
*/
for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
stmt_ty s = asdl_seq_GET(mod->v.Module.body, i);
if (done && s->lineno > prev_line)
return 1;
prev_line = s->lineno;
/* The tests below will return from this function unless it is
still possible to find a future statement. The only things
that can precede a future statement are another future
statement and a doc string.
*/
if (s->kind == ImportFrom_kind) {
if (s->v.ImportFrom.module == future) {
if (done) {
PyErr_SetString(PyExc_SyntaxError,
ERR_LATE_FUTURE);
PyErr_SyntaxLocation(filename,
s->lineno);
return 0;
}
if (!future_check_features(ff, s, filename))
return 0;
ff->ff_lineno = s->lineno;
}
else
done = 1;
}
ff->ff_last_lineno = n->n_lineno;
break;
case atom:
if (TYPE(CHILD(n, 0)) == STRING
&& ff->ff_found_docstring == 0) {
ff->ff_found_docstring = 1;
return 0;
else if (s->kind == Expr_kind && !found_docstring) {
expr_ty e = s->v.Expr.value;
if (e->kind != Str_kind)
done = 1;
else
found_docstring = 1;
}
ff->ff_last_lineno = n->n_lineno;
return 0;
default:
ff->ff_last_lineno = n->n_lineno;
return 0;
else
done = 1;
}
return 0;
return 1;
}
PyFutureFeatures *
PyNode_Future(node *n, const char *filename)
PyFuture_FromAST(mod_ty mod, const char *filename)
{
PyFutureFeatures *ff;
ff = (PyFutureFeatures *)PyMem_Malloc(sizeof(PyFutureFeatures));
if (ff == NULL)
return NULL;
ff->ff_found_docstring = 0;
ff->ff_last_lineno = -1;
ff->ff_features = 0;
ff->ff_lineno = -1;
if (future_parse(ff, n, filename) < 0) {
if (!future_parse(ff, mod, filename)) {
PyMem_Free((void *)ff);
return NULL;
}

23
Python/import.c
View File

@ -3,10 +3,11 @@
#include "Python.h"
#include "node.h"
#include "token.h"
#include "Python-ast.h"
#include "pythonrun.h"
#include "errcode.h"
#include "marshal.h"
#include "code.h"
#include "compile.h"
#include "eval.h"
#include "osdefs.h"
@ -766,17 +767,17 @@ load_compiled_module(char *name, char *cpathname, FILE *fp)
/* Parse a source file and return the corresponding code object */
static PyCodeObject *
parse_source_module(char *pathname, FILE *fp)
parse_source_module(const char *pathname, FILE *fp)
{
PyCodeObject *co;
node *n;
n = PyParser_SimpleParseFile(fp, pathname, Py_file_input);
if (n == NULL)
return NULL;
co = PyNode_Compile(n, pathname);
PyNode_Free(n);
PyCodeObject *co = NULL;
mod_ty mod;
mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0,
NULL);
if (mod) {
co = PyAST_Compile(mod, pathname, NULL);
free_mod(mod);
}
return co;
}

1
Python/marshal.c
View File

@ -6,6 +6,7 @@
#include "Python.h"
#include "longintrepr.h"
#include "code.h"
#include "compile.h"
#include "marshal.h"

276
Python/pythonrun.c
View File

@ -3,13 +3,16 @@
#include "Python.h"
#include "Python-ast.h"
#include "grammar.h"
#include "node.h"
#include "token.h"
#include "parsetok.h"
#include "errcode.h"
#include "code.h"
#include "compile.h"
#include "symtable.h"
#include "ast.h"
#include "eval.h"
#include "marshal.h"
@ -32,9 +35,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */
/* Forward */
static void initmain(void);
static void initsite(void);
static PyObject *run_err_node(node *, const char *, PyObject *, PyObject *,
static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *,
PyCompilerFlags *);
static PyObject *run_node(node *, const char *, PyObject *, PyObject *,
static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
PyCompilerFlags *);
static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
PyCompilerFlags *);
@ -634,25 +637,7 @@ initsite(void)
/* Parse input from a file and execute it */
int
PyRun_AnyFile(FILE *fp, const char *filename)
{
return PyRun_AnyFileExFlags(fp, filename, 0, NULL);
}
int
PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
{
return PyRun_AnyFileExFlags(fp, filename, 0, flags);
}
int
PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit)
{
return PyRun_AnyFileExFlags(fp, filename, closeit, NULL);
}
int
PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
PyRun_AnyFileExFlags(FILE *fp, char *filename, int closeit,
PyCompilerFlags *flags)
{
if (filename == NULL)
@ -667,12 +652,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
return PyRun_SimpleFileExFlags(fp, filename, closeit, flags);
}
int
PyRun_InteractiveLoop(FILE *fp, const char *filename)
{
return PyRun_InteractiveLoopFlags(fp, filename, NULL);
}
int
PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
{
@ -708,12 +687,6 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
}
}
int
PyRun_InteractiveOne(FILE *fp, const char *filename)
{
return PyRun_InteractiveOneFlags(fp, filename, NULL);
}
/* compute parser flags based on compiler flags */
#define PARSER_FLAGS(flags) \
(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
@ -723,9 +696,9 @@ int
PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
{
PyObject *m, *d, *v, *w;
node *n;
perrdetail err;
mod_ty mod;
char *ps1 = "", *ps2 = "";
int errcode = 0;
v = PySys_GetObject("ps1");
if (v != NULL) {
@ -743,26 +716,25 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
else if (PyString_Check(w))
ps2 = PyString_AsString(w);
}
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
Py_single_input, ps1, ps2, &err,
PARSER_FLAGS(flags));
mod = PyParser_ASTFromFile(fp, filename,
Py_single_input, ps1, ps2,
flags, &errcode);
Py_XDECREF(v);
Py_XDECREF(w);
if (n == NULL) {
if (err.error == E_EOF) {
if (err.text)
PyMem_DEL(err.text);
if (mod == NULL) {
if (errcode == E_EOF) {
PyErr_Clear();
return E_EOF;
}
err_input(&err);
PyErr_Print();
return err.error;
return -1;
}
m = PyImport_AddModule("__main__");
if (m == NULL)
return -1;
d = PyModule_GetDict(m);
v = run_node(n, filename, d, d, flags);
v = run_mod(mod, filename, d, d, flags);
free_mod(mod);
if (v == NULL) {
PyErr_Print();
return -1;
@ -773,12 +745,6 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
return 0;
}
int
PyRun_SimpleFile(FILE *fp, const char *filename)
{
return PyRun_SimpleFileEx(fp, filename, 0);
}
/* Check whether a file maybe a pyc file: Look at the extension,
the file type, and, if we may close it, at the first few bytes. */
@ -819,12 +785,6 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
return 0;
}
int
PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit)
{
return PyRun_SimpleFileExFlags(fp, filename, closeit, NULL);
}
int
PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
PyCompilerFlags *flags)
@ -873,12 +833,6 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
return 0;
}
int
PyRun_SimpleString(const char *command)
{
return PyRun_SimpleStringFlags(command, NULL);
}
int
PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
{
@ -1054,6 +1008,8 @@ PyErr_PrintEx(int set_sys_last_vars)
handle_system_exit();
}
PyErr_Fetch(&exception, &v, &tb);
if (exception == NULL)
return;
PyErr_NormalizeException(&exception, &v, &tb);
if (exception == NULL)
return;
@ -1195,74 +1151,48 @@ void PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb)
}
PyObject *
PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals)
PyRun_StringFlags(const char *str, int start, PyObject *globals,
PyObject *locals, PyCompilerFlags *flags)
{
return run_err_node(PyParser_SimpleParseString(str, start),
"<string>", globals, locals, NULL);
}
PyObject *
PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals,
PyObject *locals)
{
return PyRun_FileEx(fp, filename, start, globals, locals, 0);
}
PyObject *
PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals,
PyObject *locals, int closeit)
{
node *n = PyParser_SimpleParseFile(fp, filename, start);
if (closeit)
fclose(fp);
return run_err_node(n, filename, globals, locals, NULL);
}
PyObject *
PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals,
PyCompilerFlags *flags)
{
return run_err_node(PyParser_SimpleParseStringFlags(
str, start, PARSER_FLAGS(flags)),
"<string>", globals, locals, flags);
}
PyObject *
PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals,
PyObject *locals, PyCompilerFlags *flags)
{
return PyRun_FileExFlags(fp, filename, start, globals, locals, 0,
flags);
PyObject *ret;
mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags);
ret = run_err_mod(mod, "<string>", globals, locals, flags);
free_mod(mod);
return ret;
}
PyObject *
PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
PyObject *locals, int closeit, PyCompilerFlags *flags)
{
node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
PARSER_FLAGS(flags));
PyObject *ret;
mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
flags, NULL);
if (mod == NULL)
return NULL;
if (closeit)
fclose(fp);
return run_err_node(n, filename, globals, locals, flags);
ret = run_err_mod(mod, filename, globals, locals, flags);
free_mod(mod);
return ret;
}
static PyObject *
run_err_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
PyCompilerFlags *flags)
run_err_mod(mod_ty mod, const char *filename, PyObject *globals,
PyObject *locals, PyCompilerFlags *flags)
{
if (n == NULL)
if (mod == NULL)
return NULL;
return run_node(n, filename, globals, locals, flags);
return run_mod(mod, filename, globals, locals, flags);
}
static PyObject *
run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
PyCompilerFlags *flags)
{
PyCodeObject *co;
PyObject *v;
co = PyNode_CompileFlags(n, filename, flags);
PyNode_Free(n);
co = PyAST_Compile(mod, filename, flags);
if (co == NULL)
return NULL;
v = PyEval_EvalCode(co, globals, locals);
@ -1271,8 +1201,8 @@ run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
}
static PyObject *
run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals,
PyCompilerFlags *flags)
run_pyc_file(FILE *fp, const char *filename, PyObject *globals,
PyObject *locals, PyCompilerFlags *flags)
{
PyCodeObject *co;
PyObject *v;
@ -1302,42 +1232,78 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals
return v;
}
PyObject *
Py_CompileString(const char *str, const char *filename, int start)
{
return Py_CompileStringFlags(str, filename, start, NULL);
}
PyObject *
Py_CompileStringFlags(const char *str, const char *filename, int start,
PyCompilerFlags *flags)
{
node *n;
mod_ty mod;
PyCodeObject *co;
n = PyParser_SimpleParseStringFlagsFilename(str, filename, start,
PARSER_FLAGS(flags));
if (n == NULL)
mod = PyParser_ASTFromString(str, filename, start, flags);
if (mod == NULL)
return NULL;
co = PyNode_CompileFlags(n, filename, flags);
PyNode_Free(n);
co = PyAST_Compile(mod, filename, flags);
free_mod(mod);
return (PyObject *)co;
}
struct symtable *
Py_SymtableString(const char *str, const char *filename, int start)
{
node *n;
mod_ty mod;
struct symtable *st;
n = PyParser_SimpleParseStringFlagsFilename(str, filename,
start, 0);
if (n == NULL)
mod = PyParser_ASTFromString(str, filename, start, NULL);
if (mod == NULL)
return NULL;
st = PyNode_CompileSymtable(n, filename);
PyNode_Free(n);
st = PySymtable_Build(mod, filename, 0);
free_mod(mod);
return st;
}
/* Preferred access to parser is through AST. */
mod_ty
PyParser_ASTFromString(const char *s, const char *filename, int start,
PyCompilerFlags *flags)
{
node *n;
mod_ty mod;
perrdetail err;
n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar,
start, &err,
PARSER_FLAGS(flags));
if (n) {
mod = PyAST_FromNode(n, flags, filename);
PyNode_Free(n);
return mod;
}
else {
err_input(&err);
return NULL;
}
}
mod_ty
PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
char *ps2, PyCompilerFlags *flags, int *errcode)
{
node *n;
mod_ty mod;
perrdetail err;
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
ps1, ps2, &err, PARSER_FLAGS(flags));
if (n) {
mod = PyAST_FromNode(n, flags, filename);
PyNode_Free(n);
return mod;
}
else {
err_input(&err);
if (errcode)
*errcode = err.error;
return NULL;
}
}
/* Simplified interface to parsefile -- return node or set exception */
node *
@ -1349,15 +1315,10 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla
(char *)0, (char *)0, &err, flags);
if (n == NULL)
err_input(&err);
return n;
}
node *
PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
{
return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
}
/* Simplified interface to parsestring -- return node or set exception */
node *
@ -1372,12 +1333,6 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags)
return n;
}
node *
PyParser_SimpleParseString(const char *str, int start)
{
return PyParser_SimpleParseStringFlags(str, start, 0);
}
node *
PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
int start, int flags)
@ -1418,12 +1373,6 @@ err_input(perrdetail *err)
PyObject* u = NULL;
char *msg = NULL;
errtype = PyExc_SyntaxError;
v = Py_BuildValue("(ziiz)", err->filename,
err->lineno, err->offset, err->text);
if (err->text != NULL) {
PyMem_DEL(err->text);
err->text = NULL;
}
switch (err->error) {
case E_SYNTAX:
errtype = PyExc_IndentationError;
@ -1450,11 +1399,9 @@ err_input(perrdetail *err)
case E_INTR:
if (!PyErr_Occurred())
PyErr_SetNone(PyExc_KeyboardInterrupt);
Py_XDECREF(v);
return;
case E_NOMEM:
PyErr_NoMemory();
Py_XDECREF(v);
return;
case E_EOF:
msg = "unexpected EOF while parsing";
@ -1498,7 +1445,15 @@ err_input(perrdetail *err)
msg = "unknown parsing error";
break;
}
w = Py_BuildValue("(sO)", msg, v);
v = Py_BuildValue("(ziiz)", err->filename,
err->lineno, err->offset, err->text);
if (err->text != NULL) {
PyMem_DEL(err->text);
err->text = NULL;
}
w = NULL;
if (v != NULL)
w = Py_BuildValue("(sO)", msg, v);
Py_XDECREF(u);
Py_XDECREF(v);
PyErr_SetObject(errtype, w);
@ -1687,3 +1642,20 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler)
return oldhandler;
#endif
}
/* Deprecated C API functions still provided for binary compatiblity */
#undef PyParser_SimpleParseFile
#undef PyParser_SimpleParseString
node *
PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
{
return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
}
node *
PyParser_SimpleParseString(const char *str, int start)
{
return PyParser_SimpleParseStringFlags(str, start, 0);
}

1241
Python/symtable.c
View File

File diff suppressed because it is too large Load Diff

2
Python/sysmodule.c
View File

@ -15,7 +15,7 @@ Data members:
*/
#include "Python.h"
#include "compile.h"
#include "code.h"
#include "frameobject.h"
#include "eval.h"

2
Python/traceback.c
View File

@ -3,7 +3,7 @@
#include "Python.h"
#include "compile.h"
#include "code.h"
#include "frameobject.h"
#include "structmember.h"
#include "osdefs.h"

11
Tools/compiler/dumppyc.py
View File

@ -28,7 +28,7 @@ def walk(co, match=None):
if type(obj) == types.CodeType:
walk(obj, match)
def main(filename, codename=None):
def load(filename, codename=None):
co = loadCode(filename)
walk(co, codename)
@ -39,6 +39,9 @@ if __name__ == "__main__":
else:
filename = sys.argv[1]
codename = None
if filename.endswith('.py') and os.path.exists(filename+"c"):
filename += "c"
main(filename, codename)
if filename.endswith('.py'):
buf = open(filename).read()
co = compile(buf, filename, "exec")
walk(co)
else:
load(filename, codename)