Merging the gen-branch into the main line, at Guido's direction. Yay!

Bugfix candidate in inspect.py:  it was referencing "self" outside of
a method.

Grammar/Grammar

@@ -43,10 +43,11 @@ augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>='
 print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] )
 del_stmt: 'del' exprlist
 pass_stmt: 'pass'
-flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
 break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
+yield_stmt: 'yield' testlist
 raise_stmt: 'raise' [test [',' test [',' test]]]
 import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
 import_as_name: NAME [NAME NAME]

Include/compile.h

@@ -33,6 +33,7 @@ typedef struct {
 #define CO_VARARGS	0x0004
 #define CO_VARKEYWORDS	0x0008
 #define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
 
 extern DL_IMPORT(PyTypeObject) PyCode_Type;
 

Include/frameobject.h

@@ -21,6 +21,8 @@ typedef struct _frame {
     PyObject *f_globals;	/* global symbol table (PyDictObject) */
     PyObject *f_locals;		/* local symbol table (PyDictObject) */
     PyObject **f_valuestack;	/* points after the last local */
+    PyObject **f_stackbottom;   /* points to the last item on the stack if
+                                  frame has yielded. */
     PyObject *f_trace;		/* Trace function */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
     PyThreadState *f_tstate;

Include/graminit.h

@@ -18,47 +18,48 @@
 #define break_stmt 273
 #define continue_stmt 274
 #define return_stmt 275
-#define raise_stmt 276
+#define yield_stmt 276
-#define import_stmt 277
+#define raise_stmt 277
-#define import_as_name 278
+#define import_stmt 278
-#define dotted_as_name 279
+#define import_as_name 279
-#define dotted_name 280
+#define dotted_as_name 280
-#define global_stmt 281
+#define dotted_name 281
-#define exec_stmt 282
+#define global_stmt 282
-#define assert_stmt 283
+#define exec_stmt 283
-#define compound_stmt 284
+#define assert_stmt 284
-#define if_stmt 285
+#define compound_stmt 285
-#define while_stmt 286
+#define if_stmt 286
-#define for_stmt 287
+#define while_stmt 287
-#define try_stmt 288
+#define for_stmt 288
-#define except_clause 289
+#define try_stmt 289
-#define suite 290
+#define except_clause 290
-#define test 291
+#define suite 291
-#define and_test 292
+#define test 292
-#define not_test 293
+#define and_test 293
-#define comparison 294
+#define not_test 294
-#define comp_op 295
+#define comparison 295
-#define expr 296
+#define comp_op 296
-#define xor_expr 297
+#define expr 297
-#define and_expr 298
+#define xor_expr 298
-#define shift_expr 299
+#define and_expr 299
-#define arith_expr 300
+#define shift_expr 300
-#define term 301
+#define arith_expr 301
-#define factor 302
+#define term 302
-#define power 303
+#define factor 303
-#define atom 304
+#define power 304
-#define listmaker 305
+#define atom 305
-#define lambdef 306
+#define listmaker 306
-#define trailer 307
+#define lambdef 307
-#define subscriptlist 308
+#define trailer 308
-#define subscript 309
+#define subscriptlist 309
-#define sliceop 310
+#define subscript 310
-#define exprlist 311
+#define sliceop 311
-#define testlist 312
+#define exprlist 312
-#define dictmaker 313
+#define testlist 313
-#define classdef 314
+#define dictmaker 314
-#define arglist 315
+#define classdef 315
-#define argument 316
+#define arglist 316
-#define list_iter 317
+#define argument 317
-#define list_for 318
+#define list_iter 318
-#define list_if 319
+#define list_for 319
+#define list_if 320

Include/opcode.h

@@ -71,6 +71,7 @@ extern "C" {
 #define RETURN_VALUE	83
 #define IMPORT_STAR	84
 #define EXEC_STMT	85
+#define YIELD_VALUE	86
 
 #define POP_BLOCK	87
 #define END_FINALLY	88

Include/symtable.h

@@ -46,6 +46,7 @@ typedef struct _symtable_entry {
 	int ste_nested;          /* true if scope is nested */
 	int ste_child_free;      /* true if a child scope has free variables,
 				    including free refs to globals */
+	int ste_generator;       /* true if namespace is a generator */
 	int ste_opt_lineno;      /* lineno of last exec or import * */
 	struct symtable *ste_table;
 } PySymtableEntryObject;

Lib/dis.py

@@ -223,6 +223,7 @@ def_op('LOAD_LOCALS', 82)
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('EXEC_STMT', 85)
+def_op('YIELD_STMT', 86)
 
 def_op('POP_BLOCK', 87)
 def_op('END_FINALLY', 88)

Lib/inspect.py

@@ -349,32 +349,28 @@ class ListReader:
             return self.lines[i]
         else: return ''
 
-class EndOfBlock(Exception): pass
-
-class BlockFinder:
-    """Provide a tokeneater() method to detect the end of a code block."""
-    def __init__(self):
-        self.indent = 0
-        self.started = 0
-        self.last = 0
-
-    def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
-        if not self.started:
-            if type == tokenize.NAME: self.started = 1
-        elif type == tokenize.NEWLINE:
-            self.last = srow
-        elif type == tokenize.INDENT:
-            self.indent = self.indent + 1
-        elif type == tokenize.DEDENT:
-            self.indent = self.indent - 1
-            if self.indent == 0: raise EndOfBlock, self.last
-
 def getblock(lines):
     """Extract the block of code at the top of the given list of lines."""
-    try:
+
-        tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
+    indent = 0
-    except EndOfBlock, eob:
+    started = 0
-        return lines[:eob.args[0]]
+    last = 0
+    tokens = tokenize.generate_tokens(ListReader(lines).readline)
+
+    for (type, token, (srow, scol), (erow, ecol), line) in tokens:
+        if not started:
+            if type == tokenize.NAME:
+                started = 1
+        elif type == tokenize.NEWLINE:
+            last = srow
+        elif type == tokenize.INDENT:
+            indent = indent + 1
+        elif type == tokenize.DEDENT:
+            indent = indent - 1
+            if indent == 0:
+                return lines[:last]
+    else:
+        raise ValueError, "unable to find block"
 
 def getsourcelines(object):
     """Return a list of source lines and starting line number for an object.

Lib/tabnanny.py

@@ -77,9 +77,8 @@ def check(file):
     if verbose > 1:
         print "checking", `file`, "..."
 
-    reset_globals()
     try:
-        tokenize.tokenize(f.readline, tokeneater)
+        process_tokens(tokenize.generate_tokens(f.readline))
 
     except tokenize.TokenError, msg:
         errprint("%s: Token Error: %s" % (`file`, str(msg)))
@@ -244,28 +243,19 @@ def format_witnesses(w):
         prefix = prefix + "s"
     return prefix + " " + string.join(firsts, ', ')
 
-# The collection of globals, the reset_globals() function, and the
+# Need Guido's enhancement
-# tokeneater() function, depend on which version of tokenize is
+assert hasattr(tokenize, 'NL'), "tokenize module too old"
-# in use.
 
-if hasattr(tokenize, 'NL'):
+def process_tokens(tokens,
-    # take advantage of Guido's patch!
-
-    indents = []
-    check_equal = 0
-
-    def reset_globals():
-        global indents, check_equal
-        check_equal = 0
-        indents = [Whitespace("")]
-
-    def tokeneater(type, token, start, end, line,
                    INDENT=tokenize.INDENT,
                    DEDENT=tokenize.DEDENT,
                    NEWLINE=tokenize.NEWLINE,
-                   JUNK=(tokenize.COMMENT, tokenize.NL) ):
+                   JUNK=(tokenize.COMMENT, tokenize.NL)):
-        global indents, check_equal
 
+    indents = [Whitespace("")]
+    check_equal = 0
+
+    for (type, token, start, end, line) in tokens:
         if type == NEWLINE:
             # a program statement, or ENDMARKER, will eventually follow,
             # after some (possibly empty) run of tokens of the form
@@ -311,62 +301,6 @@ if hasattr(tokenize, 'NL'):
                 msg = "indent not equal e.g. " + format_witnesses(witness)
                 raise NannyNag(start[0], msg, line)
 
-else:
-    # unpatched version of tokenize
-
-    nesting_level = 0
-    indents = []
-    check_equal = 0
-
-    def reset_globals():
-        global nesting_level, indents, check_equal
-        nesting_level = check_equal = 0
-        indents = [Whitespace("")]
-
-    def tokeneater(type, token, start, end, line,
-                   INDENT=tokenize.INDENT,
-                   DEDENT=tokenize.DEDENT,
-                   NEWLINE=tokenize.NEWLINE,
-                   COMMENT=tokenize.COMMENT,
-                   OP=tokenize.OP):
-        global nesting_level, indents, check_equal
-
-        if type == INDENT:
-            check_equal = 0
-            thisguy = Whitespace(token)
-            if not indents[-1].less(thisguy):
-                witness = indents[-1].not_less_witness(thisguy)
-                msg = "indent not greater e.g. " + format_witnesses(witness)
-                raise NannyNag(start[0], msg, line)
-            indents.append(thisguy)
-
-        elif type == DEDENT:
-            del indents[-1]
-
-        elif type == NEWLINE:
-            if nesting_level == 0:
-                check_equal = 1
-
-        elif type == COMMENT:
-            pass
-
-        elif check_equal:
-            check_equal = 0
-            thisguy = Whitespace(line)
-            if not indents[-1].equal(thisguy):
-                witness = indents[-1].not_equal_witness(thisguy)
-                msg = "indent not equal e.g. " + format_witnesses(witness)
-                raise NannyNag(start[0], msg, line)
-
-        if type == OP and token in ('{', '[', '('):
-            nesting_level = nesting_level + 1
-
-        elif type == OP and token in ('}', ']', ')'):
-            if nesting_level == 0:
-                raise NannyNag(start[0],
-                               "unbalanced bracket '" + token + "'",
-                               line)
-            nesting_level = nesting_level - 1
 
 if __name__ == '__main__':
     main()

Lib/tokenize.py

@@ -111,7 +111,12 @@ def tokenize(readline, tokeneater=printtoken):
     except StopTokenizing:
         pass
 
+# backwards compatible interface, probably not used
 def tokenize_loop(readline, tokeneater):
+    for token_info in generate_tokens(readline):
+        apply(tokeneater, token_info)
+
+def generate_tokens(readline):
     lnum = parenlev = continued = 0
     namechars, numchars = string.letters + '_', string.digits
     contstr, needcont = '', 0
@@ -129,12 +134,12 @@ def tokenize_loop(readline, tokeneater):
             endmatch = endprog.match(line)
             if endmatch:
                 pos = end = endmatch.end(0)
-                tokeneater(STRING, contstr + line[:end],
+                yield (STRING, contstr + line[:end],
                            strstart, (lnum, end), contline + line)
                 contstr, needcont = '', 0
                 contline = None
             elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
-                tokeneater(ERRORTOKEN, contstr + line,
+                yield (ERRORTOKEN, contstr + line,
                            strstart, (lnum, len(line)), contline)
                 contstr = ''
                 contline = None
@@ -156,16 +161,16 @@ def tokenize_loop(readline, tokeneater):
             if pos == max: break
 
             if line[pos] in '#\r\n':           # skip comments or blank lines
-                tokeneater((NL, COMMENT)[line[pos] == '#'], line[pos:],
+                yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
             if column > indents[-1]:           # count indents or dedents
                 indents.append(column)
-                tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
             while column < indents[-1]:
                 indents = indents[:-1]
-                tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line)
+                yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
 
         else:                                  # continued statement
             if not line:
@@ -181,12 +186,12 @@ def tokenize_loop(readline, tokeneater):
 
                 if initial in numchars or \
                    (initial == '.' and token != '.'):      # ordinary number
-                    tokeneater(NUMBER, token, spos, epos, line)
+                    yield (NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
-                    tokeneater(parenlev > 0 and NL or NEWLINE,
+                    yield (parenlev > 0 and NL or NEWLINE,
                                token, spos, epos, line)
                 elif initial == '#':
-                    tokeneater(COMMENT, token, spos, epos, line)
+                    yield (COMMENT, token, spos, epos, line)
                 elif token in ("'''", '"""',               # triple-quoted
                                "r'''", 'r"""', "R'''", 'R"""',
                                "u'''", 'u"""', "U'''", 'U"""',
@@ -197,7 +202,7 @@ def tokenize_loop(readline, tokeneater):
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
                         token = line[start:pos]
-                        tokeneater(STRING, token, spos, (lnum, pos), line)
+                        yield (STRING, token, spos, (lnum, pos), line)
                     else:
                         strstart = (lnum, start)           # multiple lines
                         contstr = line[start:]
@@ -216,23 +221,23 @@ def tokenize_loop(readline, tokeneater):
                         contline = line
                         break
                     else:                                  # ordinary string
-                        tokeneater(STRING, token, spos, epos, line)
+                        yield (STRING, token, spos, epos, line)
                 elif initial in namechars:                 # ordinary name
-                    tokeneater(NAME, token, spos, epos, line)
+                    yield (NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     continued = 1
                 else:
                     if initial in '([{': parenlev = parenlev + 1
                     elif initial in ')]}': parenlev = parenlev - 1
-                    tokeneater(OP, token, spos, epos, line)
+                    yield (OP, token, spos, epos, line)
             else:
-                tokeneater(ERRORTOKEN, line[pos],
+                yield (ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos = pos + 1
 
     for indent in indents[1:]:                 # pop remaining indent levels
-        tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '')
+        yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
-    tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+    yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
 
 if __name__ == '__main__':                     # testing
     import sys

Objects/frameobject.c

@@ -67,6 +67,7 @@ frame_dealloc(PyFrameObject *f)
 {
 	int i, slots;
 	PyObject **fastlocals;
+	PyObject **p;
 
 	Py_TRASHCAN_SAFE_BEGIN(f)
 	/* Kill all local variables */
@@ -76,6 +77,10 @@ frame_dealloc(PyFrameObject *f)
 		Py_XDECREF(*fastlocals);
 	}
 
+	/* Free stack */
+	for (p = f->f_valuestack; p < f->f_stackbottom; p++) {
+		Py_XDECREF(*p);
+	}
 	Py_XDECREF(f->f_back);
 	Py_XDECREF(f->f_code);
 	Py_XDECREF(f->f_builtins);
@@ -221,6 +226,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
 		f->f_localsplus[extras] = NULL;
 
 	f->f_valuestack = f->f_localsplus + (f->f_nlocals + ncells + nfrees);
+	f->f_stackbottom = f->f_valuestack;
 
 	return f;
 }

Python/ceval.c

@@ -40,6 +40,7 @@ static PyObject *eval_code2(PyCodeObject *,
 			    PyObject **, int,
 			    PyObject *);
 
+static PyObject *eval_frame(PyFrameObject *);
 static char *get_func_name(PyObject *);
 static char *get_func_desc(PyObject *);
 static PyObject *call_object(PyObject *, PyObject *, PyObject *);
@@ -106,6 +107,124 @@ static PyObject *str_line = NULL;
 static PyObject *str_return = NULL;
 
 
+staticforward PyTypeObject gentype;
+
+typedef struct {
+	PyObject_HEAD
+	PyFrameObject *frame;
+        int running; /* true if generator is being executed */ 
+} genobject;
+
+static PyObject *
+gen_new(PyFrameObject *f)
+{
+	genobject *gen = PyObject_New(genobject, &gentype);
+	if (gen == NULL) {
+		Py_DECREF(f);
+		return NULL;
+	}
+	gen->frame = f;
+	gen->running = 0;
+	return (PyObject *)gen;
+}
+
+static void
+gen_dealloc(genobject *gen)
+{
+	Py_DECREF(gen->frame);
+	PyObject_DEL(gen);
+}
+
+static PyObject *
+gen_iternext(genobject *gen)
+{
+	PyFrameObject *f = gen->frame;
+	PyObject *result;
+
+	if (gen->running) {
+		PyErr_SetString(PyExc_ValueError,
+				"generator already executing");
+		return NULL;
+	}
+	if (f->f_stackbottom == NULL) {
+		return NULL;
+	}
+        gen->running = 1;
+	result = eval_frame(f);
+        gen->running = 0;
+        return result;
+}
+
+static PyObject *
+gen_next(genobject *gen, PyObject *args)
+{
+	PyObject *result;
+
+	if (!PyArg_ParseTuple(args, ":next"))
+		return NULL;
+
+        result = gen_iternext(gen);
+
+        if (result == NULL && !PyErr_Occurred()) {
+		PyErr_SetObject(PyExc_StopIteration, Py_None);
+		return NULL;
+        }
+
+	return result;
+}
+
+static PyObject *
+gen_getiter(PyObject *gen)
+{
+	Py_INCREF(gen);
+	return gen;
+}
+
+static struct PyMethodDef gen_methods[] = {
+	{"next",     (PyCFunction)gen_next, METH_VARARGS,
+	 "next() -- get the next value, or raise StopIteration"},
+	{NULL,          NULL}   /* Sentinel */
+};
+
+static PyObject *
+gen_getattr(genobject *gen, char *name)
+{
+	return Py_FindMethod(gen_methods, (PyObject *)gen, name);
+}
+
+statichere PyTypeObject gentype = {
+	PyObject_HEAD_INIT(&PyType_Type)
+	0,					/* ob_size */
+	"generator",				/* tp_name */
+	sizeof(genobject),			/* tp_basicsize */
+	0,					/* tp_itemsize */
+	/* methods */
+	(destructor)gen_dealloc, 		/* tp_dealloc */
+	0,					/* tp_print */
+	(getattrfunc)gen_getattr,		/* tp_getattr */
+	0,					/* tp_setattr */
+	0,					/* tp_compare */
+	0,					/* tp_repr */
+	0,					/* tp_as_number */
+	0,					/* tp_as_sequence */
+	0,					/* tp_as_mapping */
+	0,					/* tp_hash */
+	0,					/* tp_call */
+	0,					/* tp_str */
+	0,					/* tp_getattro */
+	0,					/* tp_setattro */
+	0,					/* tp_as_buffer */
+	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+ 	0,					/* tp_doc */
+ 	0,					/* tp_traverse */
+ 	0,					/* tp_clear */
+	0,					/* tp_richcompare */
+	0,					/* tp_weaklistoffset */
+	(getiterfunc)gen_getiter,		/* tp_iter */
+	(iternextfunc)gen_iternext,		/* tp_iternext */
+};
+
+
 #ifdef WITH_THREAD
 
 #ifndef DONT_HAVE_ERRNO_H
@@ -337,7 +456,8 @@ enum why_code {
 		WHY_RERAISE,	/* Exception re-raised by 'finally' */
 		WHY_RETURN,	/* 'return' statement */
 		WHY_BREAK,	/* 'break' statement */
-		WHY_CONTINUE	/* 'continue' statement */
+		WHY_CONTINUE,	/* 'continue' statement */
+		WHY_YIELD,	/* 'yield' operator */
 };
 
 static enum why_code do_raise(PyObject *, PyObject *, PyObject *);
@@ -358,10 +478,8 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
 
 /* Interpreter main loop */
 
-static PyObject *
+PyObject *
-eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
+eval_frame(PyFrameObject *f)
-	   PyObject **args, int argcount, PyObject **kws, int kwcount,
-	   PyObject **defs, int defcount, PyObject *closure)
 {
 #ifdef DXPAIRS
 	int lastopcode = 0;
@@ -378,17 +496,17 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	register PyObject *u;
 	register PyObject *t;
 	register PyObject *stream = NULL;    /* for PRINT opcodes */
-	register PyFrameObject *f; /* Current frame */
 	register PyObject **fastlocals, **freevars;
 	PyObject *retval = NULL;	/* Return value */
 	PyThreadState *tstate = PyThreadState_GET();
+	PyCodeObject *co;
 	unsigned char *first_instr;
 #ifdef LLTRACE
 	int lltrace;
 #endif
 #if defined(Py_DEBUG) || defined(LLTRACE)
 	/* Make it easier to find out where we are with a debugger */
-	char *filename = PyString_AsString(co->co_filename);
+	char *filename;
 #endif
 
 /* Code access macros */
@@ -426,6 +544,9 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 
 /* Start of code */
 
+	if (f == NULL)
+		return NULL;
+
 #ifdef USE_STACKCHECK
 	if (tstate->recursion_depth%10 == 0 && PyOS_CheckStack()) {
 		PyErr_SetString(PyExc_MemoryError, "Stack overflow");
@@ -433,256 +554,32 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	}
 #endif
 
-	if (globals == NULL) {
+	/* push frame */
-		PyErr_SetString(PyExc_SystemError, "eval_code2: NULL globals");
-		return NULL;
-	}
-
-#ifdef LLTRACE
-	lltrace = PyDict_GetItemString(globals, "__lltrace__") != NULL;
-#endif
-
-	f = PyFrame_New(tstate,			/*back*/
-			co,			/*code*/
-			globals, locals);
-	if (f == NULL)
-		return NULL;
-
-	tstate->frame = f;
-	fastlocals = f->f_localsplus;
-	freevars = f->f_localsplus + f->f_nlocals;
-
-	if (co->co_argcount > 0 ||
-	    co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) {
-		int i;
-		int n = argcount;
-		PyObject *kwdict = NULL;
-		if (co->co_flags & CO_VARKEYWORDS) {
-			kwdict = PyDict_New();
-			if (kwdict == NULL)
-				goto fail;
-			i = co->co_argcount;
-			if (co->co_flags & CO_VARARGS)
-				i++;
-			SETLOCAL(i, kwdict);
-		}
-		if (argcount > co->co_argcount) {
-			if (!(co->co_flags & CO_VARARGS)) {
-				PyErr_Format(PyExc_TypeError,
-				    "%.200s() takes %s %d "
-				    "%sargument%s (%d given)",
-				    PyString_AsString(co->co_name),
-				    defcount ? "at most" : "exactly",
-				    co->co_argcount,
-				    kwcount ? "non-keyword " : "",
-				    co->co_argcount == 1 ? "" : "s",
-				    argcount);
-				goto fail;
-			}
-			n = co->co_argcount;
-		}
-		for (i = 0; i < n; i++) {
-			x = args[i];
-			Py_INCREF(x);
-			SETLOCAL(i, x);
-		}
-		if (co->co_flags & CO_VARARGS) {
-			u = PyTuple_New(argcount - n);
-			if (u == NULL)
-				goto fail;
-			SETLOCAL(co->co_argcount, u);
-			for (i = n; i < argcount; i++) {
-				x = args[i];
-				Py_INCREF(x);
-				PyTuple_SET_ITEM(u, i-n, x);
-			}
-		}
-		for (i = 0; i < kwcount; i++) {
-			PyObject *keyword = kws[2*i];
-			PyObject *value = kws[2*i + 1];
-			int j;
-			if (keyword == NULL || !PyString_Check(keyword)) {
-				PyErr_Format(PyExc_TypeError,
-				    "%.200s() keywords must be strings",
-				    PyString_AsString(co->co_name));
-				goto fail;
-			}
-			/* XXX slow -- speed up using dictionary? */
-			for (j = 0; j < co->co_argcount; j++) {
-				PyObject *nm = PyTuple_GET_ITEM(
-					co->co_varnames, j);
-				int cmp = PyObject_RichCompareBool(
-					keyword, nm, Py_EQ);
-				if (cmp > 0)
-					break;
-				else if (cmp < 0)
-					goto fail;
-			}
-			/* Check errors from Compare */
-			if (PyErr_Occurred())
-				goto fail;
-			if (j >= co->co_argcount) {
-				if (kwdict == NULL) {
-					PyErr_Format(PyExc_TypeError,
-					    "%.200s() got an unexpected "
-					    "keyword argument '%.400s'",
-					    PyString_AsString(co->co_name),
-					    PyString_AsString(keyword));
-					goto fail;
-				}
-				PyDict_SetItem(kwdict, keyword, value);
-			}
-			else {
-				if (GETLOCAL(j) != NULL) {
-					PyErr_Format(PyExc_TypeError,
-					     "%.200s() got multiple "
-					     "values for keyword "
-					     "argument '%.400s'",
-					     PyString_AsString(co->co_name),
-					     PyString_AsString(keyword));
-					goto fail;
-				}
-				Py_INCREF(value);
-				SETLOCAL(j, value);
-			}
-		}
-		if (argcount < co->co_argcount) {
-			int m = co->co_argcount - defcount;
-			for (i = argcount; i < m; i++) {
-				if (GETLOCAL(i) == NULL) {
-					PyErr_Format(PyExc_TypeError,
-					    "%.200s() takes %s %d "
-					    "%sargument%s (%d given)",
-					    PyString_AsString(co->co_name),
-					    ((co->co_flags & CO_VARARGS) ||
-					     defcount) ? "at least"
-						       : "exactly",
-					    m, kwcount ? "non-keyword " : "",
-					    m == 1 ? "" : "s", i);
-					goto fail;
-				}
-			}
-			if (n > m)
-				i = n - m;
-			else
-				i = 0;
-			for (; i < defcount; i++) {
-				if (GETLOCAL(m+i) == NULL) {
-					PyObject *def = defs[i];
-					Py_INCREF(def);
-					SETLOCAL(m+i, def);
-				}
-			}
-		}
-	}
-	else {
-		if (argcount > 0 || kwcount > 0) {
-			PyErr_Format(PyExc_TypeError,
-				     "%.200s() takes no arguments (%d given)",
-				     PyString_AsString(co->co_name),
-				     argcount + kwcount);
-			goto fail;
-		}
-	}
-	/* Allocate and initialize storage for cell vars, and copy free
-	   vars into frame.  This isn't too efficient right now. */
-	if (f->f_ncells) {
-		int i = 0, j = 0, nargs, found;
-		char *cellname, *argname;
-		PyObject *c;
-
-		nargs = co->co_argcount;
-		if (co->co_flags & CO_VARARGS)
-			nargs++;
-		if (co->co_flags & CO_VARKEYWORDS)
-			nargs++;
-
-		/* Check for cells that shadow args */
-		for (i = 0; i < f->f_ncells && j < nargs; ++i) {
-			cellname = PyString_AS_STRING(
-				PyTuple_GET_ITEM(co->co_cellvars, i));
-			found = 0;
-			while (j < nargs) {
-				argname = PyString_AS_STRING(
-					PyTuple_GET_ITEM(co->co_varnames, j));
-				if (strcmp(cellname, argname) == 0) {
-					c = PyCell_New(GETLOCAL(j));
-					if (c == NULL)
-						goto fail;
-					GETLOCAL(f->f_nlocals + i) = c;
-					found = 1;
-					break;
-				}
-				j++;
-			}
-			if (found == 0) {
-				c = PyCell_New(NULL);
-				if (c == NULL)
-					goto fail;
-				SETLOCAL(f->f_nlocals + i, c);
-			}
-		}
-		/* Initialize any that are left */
-		while (i < f->f_ncells) {
-			c = PyCell_New(NULL);
-			if (c == NULL)
-				goto fail;
-			SETLOCAL(f->f_nlocals + i, c);
-			i++;
-		}
-	}
-	if (f->f_nfreevars) {
-		int i;
-		for (i = 0; i < f->f_nfreevars; ++i) {
-			PyObject *o = PyTuple_GET_ITEM(closure, i);
-			Py_INCREF(o);
-			freevars[f->f_ncells + i] = o;
-		}
-	}
-
-	if (tstate->sys_tracefunc != NULL) {
-		/* tstate->sys_tracefunc, if defined, is a function that
-		   will be called  on *every* entry to a code block.
-		   Its return value, if not None, is a function that
-		   will be called at the start of each executed line
-		   of code.  (Actually, the function must return
-		   itself in order to continue tracing.)
-		   The trace functions are called with three arguments:
-		   a pointer to the current frame, a string indicating
-		   why the function is called, and an argument which
-		   depends on the situation.  The global trace function
-		   (sys.trace) is also called whenever an exception
-		   is detected. */
-		if (call_trace(&tstate->sys_tracefunc,
-			       &f->f_trace, f, str_call,
-			       Py_None/*XXX how to compute arguments now?*/)) {
-			/* Trace function raised an error */
-			goto fail;
-		}
-	}
-
-	if (tstate->sys_profilefunc != NULL) {
-		/* Similar for sys_profilefunc, except it needn't return
-		   itself and isn't called for "line" events */
-		if (call_trace(&tstate->sys_profilefunc,
-			       (PyObject**)0, f, str_call,
-			       Py_None/*XXX*/)) {
-			goto fail;
-		}
-	}
-
 	if (++tstate->recursion_depth > recursion_limit) {
 		--tstate->recursion_depth;
 		PyErr_SetString(PyExc_RuntimeError,
 				"maximum recursion depth exceeded");
 		tstate->frame = f->f_back;
-		Py_DECREF(f);
 		return NULL;
 	}
 
+	f->f_back = tstate->frame;
+	tstate->frame = f;
+
+	co = f->f_code;
+	fastlocals = f->f_localsplus;
+	freevars = f->f_localsplus + f->f_nlocals;
 	_PyCode_GETCODEPTR(co, &first_instr);
-	next_instr = first_instr;
+	next_instr = first_instr + f->f_lasti;
-	stack_pointer = f->f_valuestack;
+	stack_pointer = f->f_stackbottom;
+	f->f_stackbottom = NULL;
+
+#ifdef LLTRACE
+	lltrace = PyDict_GetItemString(f->f_globals,"__lltrace__") != NULL;
+#endif
+#if defined(Py_DEBUG) || defined(LLTRACE)
+	filename = PyString_AsString(co->co_filename);
+#endif
 
 	why = WHY_NOT;
 	err = 0;
@@ -1459,6 +1356,14 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			why = WHY_RETURN;
 			break;
 
+		case YIELD_VALUE:
+			retval = POP();
+			f->f_stackbottom = stack_pointer;
+			f->f_lasti = INSTR_OFFSET();
+			why = WHY_YIELD;
+			break;
+
+
 		case EXEC_STMT:
 			w = POP();
 			v = POP();
@@ -1484,6 +1389,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			if (PyInt_Check(v)) {
 				why = (enum why_code) PyInt_AsLong(v);
 				if (why == WHY_RETURN ||
+				    why == WHY_YIELD ||
 				    why == CONTINUE_LOOP)
 					retval = POP();
 			}
@@ -2225,7 +2131,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 
 		/* Unwind stacks if a (pseudo) exception occurred */
 
-		while (why != WHY_NOT && f->f_iblock > 0) {
+		while (why != WHY_NOT && why != WHY_YIELD && f->f_iblock > 0) {
 			PyTryBlock *b = PyFrame_BlockPop(f);
 
 			if (b->b_type == SETUP_LOOP && why == WHY_CONTINUE) {
@@ -2295,16 +2201,18 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 
 	/* Pop remaining stack entries */
 
+	/*
 	while (!EMPTY()) {
 		v = POP();
 		Py_XDECREF(v);
 	}
+	*/
 
-	if (why != WHY_RETURN)
+	if (why != WHY_RETURN && why != WHY_YIELD)
 		retval = NULL;
 
 	if (f->f_trace) {
-		if (why == WHY_RETURN) {
+		if (why == WHY_RETURN || why == WHY_YIELD) {
 			if (call_trace(&f->f_trace, &f->f_trace, f,
 				       str_return, retval)) {
 				Py_XDECREF(retval);
@@ -2314,7 +2222,8 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 		}
 	}
 
-	if (tstate->sys_profilefunc && why == WHY_RETURN) {
+	if (tstate->sys_profilefunc &&
+			(why == WHY_RETURN || why == WHY_YIELD)) {
 		if (call_trace(&tstate->sys_profilefunc, (PyObject**)0,
 			       f, str_return, retval)) {
 			Py_XDECREF(retval);
@@ -2325,18 +2234,272 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 
 	reset_exc_info(tstate);
 
+	/* pop frame */
 	--tstate->recursion_depth;
-
-  fail: /* Jump here from prelude on failure */
-
-	/* Restore previous frame and release the current one */
-
 	tstate->frame = f->f_back;
-	Py_DECREF(f);
 
 	return retval;
 }
 
+static PyObject *
+eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
+	   PyObject **args, int argcount, PyObject **kws, int kwcount,
+	   PyObject **defs, int defcount, PyObject *closure)
+{
+	register PyFrameObject *f;
+	register PyObject *retval = NULL;
+	register PyObject **fastlocals, **freevars;
+	PyThreadState *tstate = PyThreadState_GET();
+	PyObject *x, *u;
+
+	if (globals == NULL) {
+		PyErr_SetString(PyExc_SystemError, "eval_code2: NULL globals");
+		return NULL;
+	}
+
+	f = PyFrame_New(tstate,			/*back*/
+			co,			/*code*/
+			globals, locals);
+	if (f == NULL)
+		return NULL;
+
+	fastlocals = f->f_localsplus;
+	freevars = f->f_localsplus + f->f_nlocals;
+
+	if (co->co_argcount > 0 ||
+	    co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) {
+		int i;
+		int n = argcount;
+		PyObject *kwdict = NULL;
+		if (co->co_flags & CO_VARKEYWORDS) {
+			kwdict = PyDict_New();
+			if (kwdict == NULL)
+				goto fail;
+			i = co->co_argcount;
+			if (co->co_flags & CO_VARARGS)
+				i++;
+			SETLOCAL(i, kwdict);
+		}
+		if (argcount > co->co_argcount) {
+			if (!(co->co_flags & CO_VARARGS)) {
+				PyErr_Format(PyExc_TypeError,
+				    "%.200s() takes %s %d "
+				    "%sargument%s (%d given)",
+				    PyString_AsString(co->co_name),
+				    defcount ? "at most" : "exactly",
+				    co->co_argcount,
+				    kwcount ? "non-keyword " : "",
+				    co->co_argcount == 1 ? "" : "s",
+				    argcount);
+				goto fail;
+			}
+			n = co->co_argcount;
+		}
+		for (i = 0; i < n; i++) {
+			x = args[i];
+			Py_INCREF(x);
+			SETLOCAL(i, x);
+		}
+		if (co->co_flags & CO_VARARGS) {
+			u = PyTuple_New(argcount - n);
+			if (u == NULL)
+				goto fail;
+			SETLOCAL(co->co_argcount, u);
+			for (i = n; i < argcount; i++) {
+				x = args[i];
+				Py_INCREF(x);
+				PyTuple_SET_ITEM(u, i-n, x);
+			}
+		}
+		for (i = 0; i < kwcount; i++) {
+			PyObject *keyword = kws[2*i];
+			PyObject *value = kws[2*i + 1];
+			int j;
+			if (keyword == NULL || !PyString_Check(keyword)) {
+				PyErr_Format(PyExc_TypeError,
+				    "%.200s() keywords must be strings",
+				    PyString_AsString(co->co_name));
+				goto fail;
+			}
+			/* XXX slow -- speed up using dictionary? */
+			for (j = 0; j < co->co_argcount; j++) {
+				PyObject *nm = PyTuple_GET_ITEM(
+					co->co_varnames, j);
+				int cmp = PyObject_RichCompareBool(
+					keyword, nm, Py_EQ);
+				if (cmp > 0)
+					break;
+				else if (cmp < 0)
+					goto fail;
+			}
+			/* Check errors from Compare */
+			if (PyErr_Occurred())
+				goto fail;
+			if (j >= co->co_argcount) {
+				if (kwdict == NULL) {
+					PyErr_Format(PyExc_TypeError,
+					    "%.200s() got an unexpected "
+					    "keyword argument '%.400s'",
+					    PyString_AsString(co->co_name),
+					    PyString_AsString(keyword));
+					goto fail;
+				}
+				PyDict_SetItem(kwdict, keyword, value);
+			}
+			else {
+				if (GETLOCAL(j) != NULL) {
+					PyErr_Format(PyExc_TypeError,
+					     "%.200s() got multiple "
+					     "values for keyword "
+					     "argument '%.400s'",
+					     PyString_AsString(co->co_name),
+					     PyString_AsString(keyword));
+					goto fail;
+				}
+				Py_INCREF(value);
+				SETLOCAL(j, value);
+			}
+		}
+		if (argcount < co->co_argcount) {
+			int m = co->co_argcount - defcount;
+			for (i = argcount; i < m; i++) {
+				if (GETLOCAL(i) == NULL) {
+					PyErr_Format(PyExc_TypeError,
+					    "%.200s() takes %s %d "
+					    "%sargument%s (%d given)",
+					    PyString_AsString(co->co_name),
+					    ((co->co_flags & CO_VARARGS) ||
+					     defcount) ? "at least"
+						       : "exactly",
+					    m, kwcount ? "non-keyword " : "",
+					    m == 1 ? "" : "s", i);
+					goto fail;
+				}
+			}
+			if (n > m)
+				i = n - m;
+			else
+				i = 0;
+			for (; i < defcount; i++) {
+				if (GETLOCAL(m+i) == NULL) {
+					PyObject *def = defs[i];
+					Py_INCREF(def);
+					SETLOCAL(m+i, def);
+				}
+			}
+		}
+	}
+	else {
+		if (argcount > 0 || kwcount > 0) {
+			PyErr_Format(PyExc_TypeError,
+				     "%.200s() takes no arguments (%d given)",
+				     PyString_AsString(co->co_name),
+				     argcount + kwcount);
+			goto fail;
+		}
+	}
+	/* Allocate and initialize storage for cell vars, and copy free
+	   vars into frame.  This isn't too efficient right now. */
+	if (f->f_ncells) {
+		int i = 0, j = 0, nargs, found;
+		char *cellname, *argname;
+		PyObject *c;
+
+		nargs = co->co_argcount;
+		if (co->co_flags & CO_VARARGS)
+			nargs++;
+		if (co->co_flags & CO_VARKEYWORDS)
+			nargs++;
+
+		/* Check for cells that shadow args */
+		for (i = 0; i < f->f_ncells && j < nargs; ++i) {
+			cellname = PyString_AS_STRING(
+				PyTuple_GET_ITEM(co->co_cellvars, i));
+			found = 0;
+			while (j < nargs) {
+				argname = PyString_AS_STRING(
+					PyTuple_GET_ITEM(co->co_varnames, j));
+				if (strcmp(cellname, argname) == 0) {
+					c = PyCell_New(GETLOCAL(j));
+					if (c == NULL)
+						goto fail;
+					GETLOCAL(f->f_nlocals + i) = c;
+					found = 1;
+					break;
+				}
+				j++;
+			}
+			if (found == 0) {
+				c = PyCell_New(NULL);
+				if (c == NULL)
+					goto fail;
+				SETLOCAL(f->f_nlocals + i, c);
+			}
+		}
+		/* Initialize any that are left */
+		while (i < f->f_ncells) {
+			c = PyCell_New(NULL);
+			if (c == NULL)
+				goto fail;
+			SETLOCAL(f->f_nlocals + i, c);
+			i++;
+		}
+	}
+	if (f->f_nfreevars) {
+		int i;
+		for (i = 0; i < f->f_nfreevars; ++i) {
+			PyObject *o = PyTuple_GET_ITEM(closure, i);
+			Py_INCREF(o);
+			freevars[f->f_ncells + i] = o;
+		}
+	}
+
+	if (tstate->sys_tracefunc != NULL) {
+		/* tstate->sys_tracefunc, if defined, is a function that
+		   will be called  on *every* entry to a code block.
+		   Its return value, if not None, is a function that
+		   will be called at the start of each executed line
+		   of code.  (Actually, the function must return
+		   itself in order to continue tracing.)
+		   The trace functions are called with three arguments:
+		   a pointer to the current frame, a string indicating
+		   why the function is called, and an argument which
+		   depends on the situation.  The global trace function
+		   (sys.trace) is also called whenever an exception
+		   is detected. */
+		if (call_trace(&tstate->sys_tracefunc,
+			       &f->f_trace, f, str_call,
+			       Py_None/*XXX how to compute arguments now?*/)) {
+			/* Trace function raised an error */
+			goto fail;
+		}
+	}
+
+	if (tstate->sys_profilefunc != NULL) {
+		/* Similar for sys_profilefunc, except it needn't return
+		   itself and isn't called for "line" events */
+		if (call_trace(&tstate->sys_profilefunc,
+			       (PyObject**)0, f, str_call,
+			       Py_None/*XXX*/)) {
+			goto fail;
+		}
+	}
+
+	if (co->co_flags & CO_GENERATOR) {
+                /* create a new generator that owns the ready to run frame
+                 * and return that as the value */
+		return gen_new(f);
+	}
+
+        retval = eval_frame(f);
+
+  fail: /* Jump here from prelude on failure */
+
+        Py_DECREF(f);
+	return retval;
+}
+
+
 static void
 set_exc_info(PyThreadState *tstate,
 	     PyObject *type, PyObject *value, PyObject *tb)
@@ -2701,7 +2864,6 @@ _PyTrace_Init(void)
 	return 0;
 }
 
-
 PyObject *
 PyEval_GetBuiltins(void)
 {

Python/compile.c

@@ -2634,13 +2634,37 @@ com_return_stmt(struct compiling *c, node *n)
 	if (!c->c_infunction) {
 		com_error(c, PyExc_SyntaxError, "'return' outside function");
 	}
-	if (NCH(n) < 2) {
+	if (c->c_flags & CO_GENERATOR) {
-		com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+		if (NCH(n) > 1) {
+			com_error(c, PyExc_SyntaxError,
+				  "'return' with argument inside generator");
+		}
+		com_addoparg(c, LOAD_CONST,
+				com_addconst(c, PyExc_StopIteration));
 		com_push(c, 1);
+		com_addoparg(c, RAISE_VARARGS, 1);
 	}
-	else
+	else {
-		com_node(c, CHILD(n, 1));
+		if (NCH(n) < 2) {
-	com_addbyte(c, RETURN_VALUE);
+			com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+			com_push(c, 1);
+		}
+		else
+			com_node(c, CHILD(n, 1));
+		com_addbyte(c, RETURN_VALUE);
+	}
+	com_pop(c, 1);
+}
+
+static void
+com_yield_stmt(struct compiling *c, node *n)
+{
+	REQ(n, yield_stmt); /* 'yield' testlist */
+	if (!c->c_infunction) {
+		com_error(c, PyExc_SyntaxError, "'yield' outside function");
+	}
+	com_node(c, CHILD(n, 1));
+	com_addbyte(c, YIELD_VALUE);
 	com_pop(c, 1);
 }
 
@@ -3455,6 +3479,9 @@ com_node(struct compiling *c, node *n)
 	case return_stmt:
 		com_return_stmt(c, n);
 		break;
+	case yield_stmt:
+		com_yield_stmt(c, n);
+		break;
 	case raise_stmt:
 		com_raise_stmt(c, n);
 		break;
@@ -3674,10 +3701,19 @@ compile_funcdef(struct compiling *c, node *n)
 	c->c_infunction = 1;
 	com_node(c, CHILD(n, 4));
 	c->c_infunction = 0;
-	com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+	if (c->c_flags & CO_GENERATOR) {
-	com_push(c, 1);
+		com_addoparg(c, LOAD_CONST,
-	com_addbyte(c, RETURN_VALUE);
+				com_addconst(c, PyExc_StopIteration));
-	com_pop(c, 1);
+		com_push(c, 1);
+		com_addoparg(c, RAISE_VARARGS, 1);
+		com_pop(c, 1);
+	}
+	else {
+		com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None));
+		com_push(c, 1);
+		com_addbyte(c, RETURN_VALUE);
+		com_pop(c, 1);
+	}
 }
 
 static void
@@ -4342,6 +4378,8 @@ symtable_update_flags(struct compiling *c, PySymtableEntryObject *ste,
 {
 	if (c->c_future && c->c_future->ff_nested_scopes)
 		c->c_flags |= CO_NESTED;
+	if (ste->ste_generator)
+		c->c_flags |= CO_GENERATOR;
 	if (ste->ste_type != TYPE_MODULE)
 		c->c_flags |= CO_NEWLOCALS;
 	if (ste->ste_type == TYPE_FUNCTION) {
@@ -4900,6 +4938,10 @@ symtable_node(struct symtable *st, node *n)
 	case del_stmt:
 		symtable_assign(st, CHILD(n, 1), 0);
 		break;
+	case yield_stmt:
+		st->st_cur->ste_generator = 1;
+		n = CHILD(n, 1);
+		goto loop;
 	case expr_stmt:
 		if (NCH(n) == 1)
 			n = CHILD(n, 0);

Python/marshal.c

@@ -17,6 +17,7 @@
 
 #define TYPE_NULL	'0'
 #define TYPE_NONE	'N'
+#define TYPE_STOPITER	'S'
 #define TYPE_ELLIPSIS   '.'
 #define TYPE_INT	'i'
 #define TYPE_INT64	'I'
@@ -120,6 +121,9 @@ w_object(PyObject *v, WFILE *p)
 	else if (v == Py_None) {
 		w_byte(TYPE_NONE, p);
 	}
+	else if (v == PyExc_StopIteration) {
+		w_byte(TYPE_STOPITER, p);
+	}
 	else if (v == Py_Ellipsis) {
 	        w_byte(TYPE_ELLIPSIS, p);
 	}
@@ -376,6 +380,10 @@ r_object(RFILE *p)
 		Py_INCREF(Py_None);
 		return Py_None;
 
+	case TYPE_STOPITER:
+		Py_INCREF(PyExc_StopIteration);
+		return PyExc_StopIteration;
+
 	case TYPE_ELLIPSIS:
 		Py_INCREF(Py_Ellipsis);
 		return Py_Ellipsis;

Python/symtable.c

@@ -69,6 +69,7 @@ PySymtableEntry_New(struct symtable *st, char *name, int type, int lineno)
 	else
 		ste->ste_nested = 0;
 	ste->ste_child_free = 0;
+	ste->ste_generator = 0;
 
 	if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0)
 	    goto fail;