Part way to allowing "from __future__ import generators" to communicate

that info to code dynamically compiled *by* code compiled with generators
enabled.  Doesn't yet work because there's still no way to tell the parser
that "yield" is OK (unlike nested_scopes, the parser has its fingers in
this too).
Replaced PyEval_GetNestedScopes by a more-general
PyEval_MergeCompilerFlags.  Perhaps I should not have?  I doubted it was
*intended* to be part of the public API, so just did.

Include/Python.h

@@ -96,8 +96,8 @@
 #include "pystate.h"
 
 #include "modsupport.h"
-#include "ceval.h"
 #include "pythonrun.h"
+#include "ceval.h"
 #include "sysmodule.h"
 #include "intrcheck.h"
 #include "import.h"

Include/ceval.h

@@ -31,7 +31,11 @@ DL_IMPORT(PyObject *) PyEval_GetLocals(void);
 DL_IMPORT(PyObject *) PyEval_GetOwner(void);
 DL_IMPORT(PyObject *) PyEval_GetFrame(void);
 DL_IMPORT(int) PyEval_GetRestricted(void);
-DL_IMPORT(int) PyEval_GetNestedScopes(void);
+
+/* Look at the current frame's (if any) code's co_flags, and turn on
+   the corresponding compiler flags in cf->cf_flags.  Return 1 if any
+   flag was set, else return 0. */
+DL_IMPORT(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
 DL_IMPORT(int) Py_FlushLine(void);
 

Include/compile.h

@@ -34,6 +34,13 @@ typedef struct {
 #define CO_VARKEYWORDS	0x0008
 #define CO_NESTED       0x0010
 #define CO_GENERATOR    0x0020
+/* XXX Temporary hack.  Until generators are a permanent part of the
+   language, we need a way for a code object to record that generators
+   were *possible* when it was compiled.  This is so code dynamically
+   compiled *by* a code object knows whether to allow yield stmts.  In
+   effect, this passes on the "from __future__ import generators" state
+   in effect when the code block was compiled. */
+#define CO_GENERATOR_ALLOWED    0x1000
 
 extern DL_IMPORT(PyTypeObject) PyCode_Type;
 
@@ -56,6 +63,7 @@ typedef struct {
     int ff_found_docstring;
     int ff_last_lineno;
     int ff_nested_scopes;
+    int ff_generators;
 } PyFutureFeatures;
 
 DL_IMPORT(PyFutureFeatures *) PyNode_Future(struct _node *, char *);

Include/pythonrun.h

@@ -7,8 +7,13 @@
 extern "C" {
 #endif
 
+/* These flags are named after the __future__ statements that introduced
+   them.  May not remain true for later additions, so fiddle this comment
+   accordingly then. */
+#define PyCF_NESTED_SCOPES	(0x00000001UL)
+#define PyCF_GENERATORS		(0x00000002UL)
 typedef struct {
-	int cf_nested_scopes;
+	unsigned long cf_flags;  /* bitmask of PyCF_xxx flags */
 } PyCompilerFlags;
 
 DL_IMPORT(void) Py_SetProgramName(char *);

Lib/test/test_generators.py

@@ -1,3 +1,5 @@
+from __future__ import generators
+
 tutorial_tests = """
 Let's try a simple generator:
 

Modules/main.c

@@ -298,7 +298,7 @@ Py_Main(int argc, char **argv)
 			Py_DECREF(v);
 	}
 
-	cf.cf_nested_scopes = 0;
+	cf.cf_flags = 0;
 
 	if (command) {
 		sts = PyRun_SimpleString(command) != 0;

Python/bltinmodule.c

@@ -393,6 +393,7 @@ builtin_compile(PyObject *self, PyObject *args)
 	char *filename;
 	char *startstr;
 	int start;
+	PyCompilerFlags cf;
 
 	if (!PyArg_ParseTuple(args, "sss:compile", &str, &filename, &startstr))
 		return NULL;
@@ -407,11 +408,10 @@ builtin_compile(PyObject *self, PyObject *args)
 		   "compile() arg 3 must be 'exec' or 'eval' or 'single'");
 		return NULL;
 	}
-	if (PyEval_GetNestedScopes()) {
+	cf.cf_flags = 0;
-		PyCompilerFlags cf;
+	if (PyEval_MergeCompilerFlags(&cf))
-		cf.cf_nested_scopes = 1;
 		return Py_CompileStringFlags(str, filename, start, &cf);
-	} else
+	else
 		return Py_CompileString(str, filename, start);
 }
 
@@ -822,6 +822,7 @@ builtin_execfile(PyObject *self, PyObject *args)
 	PyObject *globals = Py_None, *locals = Py_None;
 	PyObject *res;
 	FILE* fp;
+	PyCompilerFlags cf;
 
 	if (!PyArg_ParseTuple(args, "s|O!O!:execfile",
 			&filename,
@@ -847,12 +848,11 @@ builtin_execfile(PyObject *self, PyObject *args)
 		PyErr_SetFromErrno(PyExc_IOError);
 		return NULL;
 	}
-	if (PyEval_GetNestedScopes()) {
+	cf.cf_flags = 0;
-		PyCompilerFlags cf;
+	if (PyEval_MergeCompilerFlags(&cf))
-		cf.cf_nested_scopes = 1;
 		res = PyRun_FileExFlags(fp, filename, Py_file_input, globals,
 				   locals, 1, &cf);
-	} else
+	else
 		res = PyRun_FileEx(fp, filename, Py_file_input, globals,
 				   locals, 1);
 	return res;

Python/ceval.c

@@ -2524,7 +2524,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	if (co->co_flags & CO_GENERATOR) {
 		/* Don't need to keep the reference to f_back, it will be set
 		 * when the generator is resumed. */
-		Py_DECREF(f->f_back);
+		Py_XDECREF(f->f_back);
 		f->f_back = NULL;
 
 		/* Create a new generator that owns the ready to run frame
@@ -2906,11 +2906,23 @@ PyEval_GetRestricted(void)
 }
 
 int
-PyEval_GetNestedScopes(void)
+PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
 {
 	PyFrameObject *current_frame = PyThreadState_Get()->frame;
-	return current_frame == NULL ? 0 : 
+	int result = 0;
-	    current_frame->f_code->co_flags & CO_NESTED;
+
+	if (current_frame != NULL) {
+		const int codeflags = current_frame->f_code->co_flags;
+		if (codeflags & CO_NESTED) {
+			result = 1;
+			cf->cf_flags |= PyCF_NESTED_SCOPES;
+		}
+		if (codeflags & CO_GENERATOR_ALLOWED) {
+			result = 1;
+			cf->cf_flags |= PyCF_GENERATORS;
+		}
+	}
+	return result;
 }
 
 int
@@ -3730,26 +3742,25 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals,
 	else if (PyFile_Check(prog)) {
 		FILE *fp = PyFile_AsFile(prog);
 		char *name = PyString_AsString(PyFile_Name(prog));
-		if (PyEval_GetNestedScopes()) {
+		PyCompilerFlags cf;
-			PyCompilerFlags cf;
+		cf.cf_flags = 0;
-			cf.cf_nested_scopes = 1;
+		if (PyEval_MergeCompilerFlags(&cf))
 			v = PyRun_FileFlags(fp, name, Py_file_input, globals,
 					    locals, &cf); 
-		} else {
+		else
 			v = PyRun_File(fp, name, Py_file_input, globals,
 				       locals); 
-		}
 	}
 	else {
 		char *str;
+		PyCompilerFlags cf;
 		if (PyString_AsStringAndSize(prog, &str, NULL))
 			return -1;
-		if (PyEval_GetNestedScopes()) {
+		cf.cf_flags = 0;
-			PyCompilerFlags cf;
+		if (PyEval_MergeCompilerFlags(&cf))
-			cf.cf_nested_scopes = 1;
 			v = PyRun_StringFlags(str, Py_file_input, globals, 
 					      locals, &cf);
-		} else
+		else
 			v = PyRun_String(str, Py_file_input, globals, locals);
 	}
 	if (plain)

Python/compile.c

@@ -3953,10 +3953,15 @@ jcompile(node *n, char *filename, struct compiling *base,
 			return NULL;
 		}
 		if (flags) {
-			if (flags->cf_nested_scopes)
+			if (flags->cf_flags & PyCF_NESTED_SCOPES)
 				sc.c_future->ff_nested_scopes = 1;
 			else if (sc.c_future->ff_nested_scopes)
-				flags->cf_nested_scopes = 1;
+				flags->cf_flags |= PyCF_NESTED_SCOPES;
+
+			if (flags->cf_flags & PyCF_GENERATORS)
+				sc.c_future->ff_generators = 1;
+			else if (sc.c_future->ff_generators)
+				flags->cf_flags |= PyCF_GENERATORS;
 		}
 		if (symtable_build(&sc, n) < 0) {
 			com_free(&sc);
@@ -4426,8 +4431,12 @@ static int
 symtable_update_flags(struct compiling *c, PySymtableEntryObject *ste,
 		      struct symbol_info *si)
 {
-	if (c->c_future && c->c_future->ff_nested_scopes)
+	if (c->c_future) {
-		c->c_flags |= CO_NESTED;
+		if (c->c_future->ff_nested_scopes)
+			c->c_flags |= CO_NESTED;
+		if (c->c_future->ff_generators)
+			c->c_flags |= CO_GENERATOR_ALLOWED;
+	}
 	if (ste->ste_generator)
 		c->c_flags |= CO_GENERATOR;
 	if (ste->ste_type != TYPE_MODULE)

Python/future.c

@@ -32,7 +32,7 @@ future_check_features(PyFutureFeatures *ff, node *n, char *filename)
 		if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
 			ff->ff_nested_scopes = 1;
 		} else if (strcmp(feature, FUTURE_GENERATORS) == 0) {
-			/* OK; this is processed by the parser */
+			ff->ff_generators= 1;
 		} else if (strcmp(feature, "braces") == 0) {
 			PyErr_SetString(PyExc_SyntaxError,
 					"not a chance");
@@ -233,6 +233,7 @@ PyNode_Future(node *n, char *filename)
 	ff->ff_found_docstring = 0;
 	ff->ff_last_lineno = -1;
 	ff->ff_nested_scopes = 0;
+	ff->ff_generators = 0;
 
 	if (future_parse(ff, n, filename) < 0) {
 		PyMem_Free((void *)ff);

Python/pythonrun.c

@@ -494,7 +494,7 @@ PyRun_InteractiveLoopFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
 
 	if (flags == NULL) {
 		flags = &local_flags;
-		local_flags.cf_nested_scopes = 0;
+		local_flags.cf_flags = 0;
 	}
 	v = PySys_GetObject("ps1");
 	if (v == NULL) {
@@ -1075,10 +1075,14 @@ run_pyc_file(FILE *fp, char *filename, PyObject *globals, PyObject *locals,
 	v = PyEval_EvalCode(co, globals, locals);
 	if (v && flags) {
 		if (co->co_flags & CO_NESTED)
-			flags->cf_nested_scopes = 1;
+			flags->cf_flags |= PyCF_NESTED_SCOPES;
+		if (co->co_flags & CO_GENERATOR)
+			flags->cf_flags |= PyCF_GENERATORS;
 #if 0
 		fprintf(stderr, "run_pyc_file: nested_scopes: %d\n",
-			flags->cf_nested_scopes);
+			flags->cf_flags & PyCF_NESTED_SCOPES);
+		fprintf(stderr, "run_pyc_file: generators: %d\n",
+			flags->cf_flags & PyCF_GENERATORS);
 #endif
 	}
 	Py_DECREF(co);