PEP 227 implementation

The majority of the changes are in the compiler.  The mainloop changes
primarily to implement the new opcodes and to pass a function's
closure to eval_code2().  Frames and functions got new slots to hold
the closure.

Include/compile.h
    Add co_freevars and co_cellvars slots to code objects.
    Update PyCode_New() to take freevars and cellvars as arguments
Include/funcobject.h
    Add func_closure slot to function objects.
    Add GetClosure()/SetClosure() functions (and corresponding
    macros) for getting at the closure.
Include/frameobject.h
    PyFrame_New() now takes a closure.
Include/opcode.h
    Add four new opcodes: MAKE_CLOSURE, LOAD_CLOSURE, LOAD_DEREF,
    STORE_DEREF.
    Remove comment about old requirement for opcodes to fit in 7
    bits.
compile.c
    Implement changes to code objects for co_freevars and co_cellvars.

    Modify symbol table to use st_cur_name (string object for the name
    of the current scope) and st_cur_children (list of nested blocks).
    Also define st_nested, which might more properly be called
    st_cur_nested.  Add several DEF_XXX flags to track def-use
    information for free variables.

    New or modified functions of note:
    com_make_closure(struct compiling *, PyCodeObject *)
        Emit LOAD_CLOSURE opcodes as needed to pass cells for free
        variables into nested scope.
    com_addop_varname(struct compiling *, int, char *)
        Emits opcodes for LOAD_DEREF and STORE_DEREF.
    get_ref_type(struct compiling *, char *name)
        Return NAME_CLOSURE if ref type is FREE or CELL
    symtable_load_symbols(struct compiling *)
        Decides what variables are cell or free based on def-use info.
        Can now raise SyntaxError if nested scopes are mixed with
        exec or from blah import *.
    make_scope_info(PyObject *, PyObject *, int, int)
        Helper functions for symtable scope stack.
    symtable_update_free_vars(struct symtable *)
        After a code block has been analyzed, it must check each of
        its children for free variables that are not defined in the
        block.  If a variable is free in a child and not defined in
        the parent, then it is defined by block the enclosing the
        current one or it is a global.  This does the right logic.
    symtable_add_use() is now a macro for symtable_add_def()
    symtable_assign(struct symtable *, node *)
        Use goto instead of for (;;)

    Fixed bug in symtable where name of keyword argument in function
    call was treated as assignment in the scope of the call site. Ex:
        def f():
            g(a=2) # a was considered a local of f

ceval.c
    eval_code2() now take one more argument, a closure.
    Implement LOAD_CLOSURE, LOAD_DEREF, STORE_DEREF, MAKE_CLOSURE>

    Also: When name error occurs for global variable, report that the
    name was global in the error mesage.

Objects/frameobject.c
    Initialize f_closure to be a tuple containing space for cellvars
    and freevars.  f_closure is NULL if neither are present.
Objects/funcobject.c
    Add support for func_closure.
Python/import.c
    Change the magic number.
Python/marshal.c
    Track changes to code objects.

Include/compile.h

@@ -18,6 +18,8 @@ typedef struct {
     PyObject *co_consts;	/* list (constants used) */
     PyObject *co_names;		/* list of strings (names used) */
     PyObject *co_varnames;	/* tuple of strings (local variable names) */
+    PyObject *co_freevars;	/* tuple of strings (free variable names) */
+    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
     /* The rest doesn't count for hash/cmp */
     PyObject *co_filename;	/* string (where it was loaded from) */
     PyObject *co_name;		/* string (name, for reference) */
@@ -42,7 +44,8 @@ struct _node; /* Declare the existence of this type */
 DL_IMPORT(PyCodeObject *) PyNode_Compile(struct _node *, char *);
 DL_IMPORT(PyCodeObject *) PyCode_New(
 	int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
-	PyObject *, PyObject *, int, PyObject *); /* same as struct above */
+	PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *); 
+        /* same as struct above */
 DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
 
 /* for internal use only */

Include/frameobject.h

@@ -20,6 +20,7 @@ typedef struct _frame {
     PyObject *f_builtins;	/* builtin symbol table (PyDictObject) */
     PyObject *f_globals;	/* global symbol table (PyDictObject) */
     PyObject *f_locals;		/* local symbol table (PyDictObject) */
+    PyObject *f_closure;        /* environment for free variables */
     PyObject **f_valuestack;	/* points after the last local */
     PyObject *f_trace;		/* Trace function */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
@@ -43,7 +44,8 @@ extern DL_IMPORT(PyTypeObject) PyFrame_Type;
 #define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
 
 DL_IMPORT(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
-                                       PyObject *, PyObject *);
+                                       PyObject *, PyObject *,
+				       PyObject *);
 
 
 /* The rest of the interface is specific for frame objects */

Include/funcobject.h

@@ -12,6 +12,7 @@ typedef struct {
     PyObject *func_code;
     PyObject *func_globals;
     PyObject *func_defaults;
+    PyObject *func_closure;
     PyObject *func_doc;
     PyObject *func_name;
     PyObject *func_dict;
@@ -26,6 +27,8 @@ extern DL_IMPORT(PyObject *) PyFunction_GetCode(PyObject *);
 extern DL_IMPORT(PyObject *) PyFunction_GetGlobals(PyObject *);
 extern DL_IMPORT(PyObject *) PyFunction_GetDefaults(PyObject *);
 extern DL_IMPORT(int) PyFunction_SetDefaults(PyObject *, PyObject *);
+extern DL_IMPORT(PyObject *) PyFunction_GetClosure(PyObject *);
+extern DL_IMPORT(int) PyFunction_SetClosure(PyObject *, PyObject *);
 
 /* Macros for direct access to these values. Type checks are *not*
    done, so use with care. */
@@ -35,6 +38,8 @@ extern DL_IMPORT(int) PyFunction_SetDefaults(PyObject *, PyObject *);
 	(((PyFunctionObject *)func) -> func_globals)
 #define PyFunction_GET_DEFAULTS(func) \
 	(((PyFunctionObject *)func) -> func_defaults)
+#define PyFunction_GET_CLOSURE(func) \
+	(((PyFunctionObject *)func) -> func_closure)
 
 #ifdef __cplusplus
 }

Include/opcode.h

@@ -114,17 +114,17 @@ extern "C" {
 
 #define SET_LINENO	127	/* Current line number */
 
-/* It used to be the case that opcodes should fit in 7 bits.  This is
-   no longer the case -- 8 bits is fine (the instruction stream is now
-   a sequence of unsigned characters).  We gladly use the new space
-   for new opcodes. */
-
 #define RAISE_VARARGS	130	/* Number of raise arguments (1, 2 or 3) */
 /* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
 #define CALL_FUNCTION	131	/* #args + (#kwargs<<8) */
 #define MAKE_FUNCTION	132	/* #defaults */
 #define BUILD_SLICE 	133	/* Number of items */
 
+#define MAKE_CLOSURE    134     /* #free vars */
+#define LOAD_CLOSURE    135     /* Load free variable from closure */
+#define LOAD_DEREF      136     /* Load and dereference from closure cell */ 
+#define STORE_DEREF     137     /* Store into cell */ 
+
 /* The next 3 opcodes must be contiguous and satisfy
    (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1  */
 #define CALL_FUNCTION_VAR          140	/* #args + (#kwargs<<8) */

Objects/frameobject.c

@@ -79,6 +79,7 @@ frame_dealloc(PyFrameObject *f)
 	Py_XDECREF(f->f_builtins);
 	Py_XDECREF(f->f_globals);
 	Py_XDECREF(f->f_locals);
+	Py_XDECREF(f->f_closure);
 	Py_XDECREF(f->f_trace);
 	Py_XDECREF(f->f_exc_type);
 	Py_XDECREF(f->f_exc_value);
@@ -106,14 +107,14 @@ PyTypeObject PyFrame_Type = {
 };
 
 PyFrameObject *
-PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
+PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, 
-            PyObject *globals, PyObject *locals)
+	    PyObject *locals, PyObject *closure)
 {
 	PyFrameObject *back = tstate->frame;
 	static PyObject *builtin_object;
 	PyFrameObject *f;
 	PyObject *builtins;
-	int extras;
+	int extras, ncells;
 
 	if (builtin_object == NULL) {
 		builtin_object = PyString_InternFromString("__builtins__");
@@ -128,6 +129,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
 		return NULL;
 	}
 	extras = code->co_stacksize + code->co_nlocals;
+	ncells = PyTuple_GET_SIZE(code->co_cellvars);
 	if (back == NULL || back->f_globals != globals) {
 		builtins = PyDict_GetItem(globals, builtin_object);
 		if (builtins != NULL && PyModule_Check(builtins))
@@ -197,6 +199,22 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
 			locals = globals;
 		Py_INCREF(locals);
 	}
+	if (closure || ncells) {
+		int i, size;
+		size = ncells;
+		if (closure)
+			size += PyTuple_GET_SIZE(closure);
+		f->f_closure = PyTuple_New(size);
+		for (i = 0; i < ncells; ++i)
+			PyTuple_SET_ITEM(f->f_closure, i, PyCell_New(NULL));
+		for (i = ncells; i < size; ++i) {
+			PyObject *o = PyTuple_GET_ITEM(closure, i - ncells);
+			Py_INCREF(o);
+			PyTuple_SET_ITEM(f->f_closure, i, o);
+		}
+	}
+	else
+		f->f_closure = NULL;
 	f->f_locals = locals;
 	f->f_trace = NULL;
 	f->f_exc_type = f->f_exc_value = f->f_exc_traceback = NULL;

Objects/funcobject.c

@@ -20,6 +20,7 @@ PyFunction_New(PyObject *code, PyObject *globals)
 		op->func_name = ((PyCodeObject *)code)->co_name;
 		Py_INCREF(op->func_name);
 		op->func_defaults = NULL; /* No default arguments */
+		op->func_closure = NULL;
 		consts = ((PyCodeObject *)code)->co_consts;
 		if (PyTuple_Size(consts) >= 1) {
 			doc = PyTuple_GetItem(consts, 0);
@@ -89,6 +90,37 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults)
 	return 0;
 }
 
+PyObject *
+PyFunction_GetClosure(PyObject *op)
+{
+	if (!PyFunction_Check(op)) {
+		PyErr_BadInternalCall();
+		return NULL;
+	}
+	return ((PyFunctionObject *) op) -> func_closure;
+}
+
+int
+PyFunction_SetClosure(PyObject *op, PyObject *closure)
+{
+	if (!PyFunction_Check(op)) {
+		PyErr_BadInternalCall();
+		return -1;
+	}
+	if (closure == Py_None)
+		closure = NULL;
+	else if (PyTuple_Check(closure)) {
+		Py_XINCREF(closure);
+	}
+	else {
+		PyErr_SetString(PyExc_SystemError, "non-tuple closure");
+		return -1;
+	}
+	Py_XDECREF(((PyFunctionObject *) op) -> func_closure);
+	((PyFunctionObject *) op) -> func_closure = closure;
+	return 0;
+}
+
 /* Methods */
 
 #define OFF(x) offsetof(PyFunctionObject, x)
@@ -98,6 +130,7 @@ static struct memberlist func_memberlist[] = {
         {"func_globals",  T_OBJECT,     OFF(func_globals),      READONLY},
         {"func_name",     T_OBJECT,     OFF(func_name),         READONLY},
         {"__name__",      T_OBJECT,     OFF(func_name),         READONLY},
+        {"func_closure",  T_OBJECT,     OFF(func_closure)},
         {"func_defaults", T_OBJECT,     OFF(func_defaults)},
         {"func_doc",      T_OBJECT,     OFF(func_doc)},
         {"__doc__",       T_OBJECT,     OFF(func_doc)},

Python/ceval.c

@@ -33,13 +33,16 @@
 
 typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *);
 
+#define REPR(ob) PyString_AS_STRING(PyObject_Repr(ob))
+
 /* Forward declarations */
 
 static PyObject *eval_code2(PyCodeObject *,
 			    PyObject *, PyObject *,
 			    PyObject **, int,
 			    PyObject **, int,
-			    PyObject **, int);
+			    PyObject **, int,
+			    PyObject *);
 
 static PyObject *call_object(PyObject *, PyObject *, PyObject *);
 static PyObject *call_cfunction(PyObject *, PyObject *, PyObject *);
@@ -78,6 +81,8 @@ static void format_exc_check_arg(PyObject *, char *, PyObject *);
 
 #define NAME_ERROR_MSG \
 	"name '%.200s' is not defined"
+#define GLOBAL_NAME_ERROR_MSG \
+	"global name '%.200s' is not defined"
 #define UNBOUNDLOCAL_ERROR_MSG \
 	"local variable '%.200s' referenced before assignment"
 
@@ -335,7 +340,8 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
 			  globals, locals,
 			  (PyObject **)NULL, 0,
 			  (PyObject **)NULL, 0,
-			  (PyObject **)NULL, 0);
+			  (PyObject **)NULL, 0,
+			  NULL);
 }
 
 
@@ -344,7 +350,7 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
 static PyObject *
 eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	   PyObject **args, int argcount, PyObject **kws, int kwcount,
-	   PyObject **defs, int defcount)
+	   PyObject **defs, int defcount, PyObject *closure)
 {
 #ifdef DXPAIRS
 	int lastopcode = 0;
@@ -425,11 +431,9 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	lltrace = PyDict_GetItemString(globals, "__lltrace__") != NULL;
 #endif
 
-	f = PyFrame_New(
+	f = PyFrame_New(tstate,			/*back*/
-			tstate,			/*back*/
 			co,			/*code*/
-			globals,		/*globals*/
+			globals, locals, closure);
-			locals);		/*locals*/
 	if (f == NULL)
 		return NULL;
 
@@ -1535,7 +1539,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			w = GETNAMEV(oparg);
 			if ((err = PyDict_DelItem(f->f_globals, w)) != 0)
 				format_exc_check_arg(
-				    PyExc_NameError, NAME_ERROR_MSG ,w);
+				    PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w);
 			break;
 
 		case LOAD_CONST:
@@ -1577,7 +1581,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 				if (x == NULL) {
 					format_exc_check_arg(
 						    PyExc_NameError,
-						    NAME_ERROR_MSG ,w);
+						    GLOBAL_NAME_ERROR_MSG ,w);
 					break;
 				}
 			}
@@ -1618,6 +1622,25 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			SETLOCAL(oparg, NULL);
 			continue;
 
+		case LOAD_CLOSURE:
+			x = PyTuple_GET_ITEM(f->f_closure, oparg);
+			Py_INCREF(x);
+			PUSH(x);
+			break;
+
+		case LOAD_DEREF:
+			x = PyTuple_GET_ITEM(f->f_closure, oparg);
+			w = PyCell_Get(x);
+			Py_INCREF(w);
+			PUSH(w);
+			break;
+
+		case STORE_DEREF:
+			w = POP();
+			x = PyTuple_GET_ITEM(f->f_closure, oparg);
+			PyCell_Set(x, w);
+			continue;
+
 		case BUILD_TUPLE:
 			x = PyTuple_New(oparg);
 			if (x != NULL) {
@@ -1939,6 +1962,46 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			PUSH(x);
 			break;
 
+		case MAKE_CLOSURE:
+		{
+			int nfree;
+			v = POP(); /* code object */
+			x = PyFunction_New(v, f->f_globals);
+			nfree = PyTuple_GET_SIZE(((PyCodeObject *)v)->co_freevars);
+			Py_DECREF(v);
+			/* XXX Maybe this should be a separate opcode? */
+			if (x != NULL && nfree > 0) {
+				v = PyTuple_New(nfree);
+				if (v == NULL) {
+					Py_DECREF(x);
+					x = NULL;
+					break;
+				}
+				while (--nfree >= 0) {
+					w = POP();
+					PyTuple_SET_ITEM(v, nfree, w);
+				}
+				err = PyFunction_SetClosure(x, v);
+				Py_DECREF(v);
+			}
+			if (x != NULL && oparg > 0) {
+				v = PyTuple_New(oparg);
+				if (v == NULL) {
+					Py_DECREF(x);
+					x = NULL;
+					break;
+				}
+				while (--oparg >= 0) {
+					w = POP();
+					PyTuple_SET_ITEM(v, oparg, w);
+				}
+				err = PyFunction_SetDefaults(x, v);
+				Py_DECREF(v);
+			}
+			PUSH(x);
+			break;
+		}
+
 		case BUILD_SLICE:
 			if (oparg == 3)
 				w = POP();
@@ -2761,8 +2824,8 @@ call_eval_code2(PyObject *func, PyObject *arg, PyObject *kw)
 		(PyCodeObject *)PyFunction_GET_CODE(func),
 		PyFunction_GET_GLOBALS(func), (PyObject *)NULL,
 		&PyTuple_GET_ITEM(arg, 0), PyTuple_Size(arg),
-		k, nk,
+		k, nk, d, nd,
-		d, nd);
+		PyFunction_GET_CLOSURE(func));
 
 	if (k != NULL)
 		PyMem_DEL(k);
@@ -2805,6 +2868,7 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
 	PyObject *co = PyFunction_GET_CODE(func);
 	PyObject *globals = PyFunction_GET_GLOBALS(func);
 	PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
+	PyObject *closure = PyFunction_GET_CLOSURE(func);
 	PyObject **d = NULL;
 	int nd = 0;
 
@@ -2814,7 +2878,8 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
 	}
 	return eval_code2((PyCodeObject *)co, globals,
 			  (PyObject *)NULL, (*pp_stack)-n, na,
-			  (*pp_stack)-2*nk, nk, d, nd);
+			  (*pp_stack)-2*nk, nk, d, nd,
+			  closure);
 }
 
 static PyObject *

Python/import.c

@@ -43,7 +43,7 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
 /* XXX Perhaps the magic number should be frozen and a version field
    added to the .pyc file header? */
 /* New way to come up with the magic number: (YEAR-1995), MONTH, DAY */
-#define MAGIC (50823 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (60124 | ((long)'\r'<<16) | ((long)'\n'<<24))
 
 /* Magic word as global; note that _PyImport_Init() can change the
    value of this global to accommodate for alterations of how the

Python/marshal.c

@@ -238,6 +238,8 @@ w_object(PyObject *v, WFILE *p)
 		w_object(co->co_consts, p);
 		w_object(co->co_names, p);
 		w_object(co->co_varnames, p);
+		w_object(co->co_freevars, p);
+		w_object(co->co_cellvars, p);
 		w_object(co->co_filename, p);
 		w_object(co->co_name, p);
 		w_short(co->co_firstlineno, p);
@@ -554,6 +556,8 @@ r_object(RFILE *p)
 			PyObject *consts = NULL;
 			PyObject *names = NULL;
 			PyObject *varnames = NULL;
+			PyObject *freevars = NULL;
+			PyObject *cellvars = NULL;
 			PyObject *filename = NULL;
 			PyObject *name = NULL;
 			int firstlineno = 0;
@@ -563,7 +567,9 @@ r_object(RFILE *p)
 			if (code) consts = r_object(p);
 			if (consts) names = r_object(p);
 			if (names) varnames = r_object(p);
-			if (varnames) filename = r_object(p);
+			if (varnames) freevars = r_object(p);
+			if (freevars) cellvars = r_object(p);
+			if (cellvars) filename = r_object(p);
 			if (filename) name = r_object(p);
 			if (name) {
 				firstlineno = r_short(p);
@@ -572,9 +578,10 @@ r_object(RFILE *p)
 			
 			if (!PyErr_Occurred()) {
 				v = (PyObject *) PyCode_New(
-					argcount, nlocals, stacksize, flags, 
+					argcount, nlocals, stacksize, flags,  
 					code, consts, names, varnames,
-					filename, name, firstlineno, lnotab);
+					freevars, cellvars, filename, name, 
+					firstlineno, lnotab);  
 			}
 			else
 				v = NULL;
@@ -582,6 +589,8 @@ r_object(RFILE *p)
 			Py_XDECREF(consts);
 			Py_XDECREF(names);
 			Py_XDECREF(varnames);
+			Py_XDECREF(freevars);
+			Py_XDECREF(cellvars);
 			Py_XDECREF(filename);
 			Py_XDECREF(name);
 			Py_XDECREF(lnotab);