Two more patches by Tony Lownds (SF# 1607548).

(1)
Combines the code paths for MAKE_FUNCTION and MAKE_CLOSURE.
Fixes a crash where functions with closures and either annotations or
keyword-only arguments result in MAKE_CLOSURE, but only
MAKE_FUNCTION has the code to handle annotations or keyword-only
arguments.
Includes enough tests to trigger the bug.

(2)
Change peepholer to not bail in the presence of EXTENDED_ARG +
MAKE_FUNCTION.
Enforce the natural 16-bit limit of annotations in compile.c.

Also update Misc/NEWS with the "input = raw_input" change.

Lib/test/test_compile.py

@@ -393,6 +393,19 @@ if 1:
         del d[..., ...]
         self.assertEqual((Ellipsis, Ellipsis) in d, False)
 
+    def test_annotation_limit(self):
+        # 16 bits are available for # of annotations, and the
+        # tuple of annotations names is counted, hence 65534
+        # is the max. Ensure the result of too many annotations is a
+        # SyntaxError.
+        s = "def f((%s)): pass"
+        s %= ', '.join('a%d:%d' % (i,i) for i in xrange(65535))        
+        self.assertRaises(SyntaxError, compile, s, '?', 'exec')
+        # Test that the max # of annotations compiles.
+        s = "def f((%s)): pass"
+        s %= ', '.join('a%d:%d' % (i,i) for i in xrange(65534))
+        compile(s, '?', 'exec')
+        
 def test_main():
     test_support.run_unittest(TestSpecifics)
 

Lib/test/test_grammar.py

@@ -322,6 +322,13 @@ class GrammarTests(unittest.TestCase):
                           {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
                            'k': 11, 'return': 12})
                            
+        # test MAKE_CLOSURE with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
     def testLambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0

Lib/test/test_peepholer.py

@@ -196,6 +196,14 @@ class TestTranforms(unittest.TestCase):
         self.assertEqual(asm.split().count('JUMP_ABSOLUTE'), 1)
         self.assertEqual(asm.split().count('RETURN_VALUE'), 2)
         
+    def test_make_function_doesnt_bail(self):
+        def f():
+            def g()->1+1: 
+                pass
+            return g
+        asm = disassemble(f)
+        self.assert_('BINARY_ADD' not in asm)              
+
 
 def test_main(verbose=None):
     import sys

Misc/NEWS

@@ -28,6 +28,10 @@ TO DO
 Core and Builtins
 -----------------
 
+- input() becomes raw_input(): the name input() now implements the
+  functionality formerly known as raw_input(); the name raw_input()
+  is no longer defined.
+
 - Objects listed in an 'except' clause must inherit from BaseException.
 
 - PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone;
@@ -82,7 +86,7 @@ Core and Builtins
   backticks (`x`), <>
 
 - Removed these Python builtins:
-  apply(), coerce(), input(), raw_input()
+  apply(), coerce()
 
 - Removed these Python methods:
   {}.has_key

Python/ceval.c

@@ -2236,6 +2236,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 		    break;
 		}
 
+		case MAKE_CLOSURE:		
 		case MAKE_FUNCTION:
 		{
 		    int posdefaults = oparg & 0xff;
@@ -2246,6 +2247,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			x = PyFunction_New(v, f->f_globals);
 			Py_DECREF(v);
 			
+			if (x != NULL && opcode == MAKE_CLOSURE) {
+				v = POP();
+				err = PyFunction_SetClosure(x, v);
+				Py_DECREF(v);
+			}
+
 			if (x != NULL && num_annotations > 0) {
 				Py_ssize_t name_ix;
 				u = POP(); /* names of args with annotations */
@@ -2308,34 +2315,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 			break;
 		}
 
-		case MAKE_CLOSURE:
-		{
-			v = POP(); /* code object */
-			x = PyFunction_New(v, f->f_globals);
-			Py_DECREF(v);
-			if (x != NULL) {
-				v = POP();
-				err = PyFunction_SetClosure(x, v);
-				Py_DECREF(v);
-			}
-			if (x != NULL && oparg > 0) {
-				v = PyTuple_New(oparg);
-				if (v == NULL) {
-					Py_DECREF(x);
-					x = NULL;
-					break;
-				}
-				while (--oparg >= 0) {
-					w = POP();
-					PyTuple_SET_ITEM(v, oparg, w);
-				}
-				err = PyFunction_SetDefaults(x, v);
-				Py_DECREF(v);
-			}
-			PUSH(x);
-			break;
-		}
-
 		case BUILD_SLICE:
 			if (oparg == 3)
 				w = POP();

Python/compile.c

@@ -836,6 +836,8 @@ opcode_stack_effect(int opcode, int oparg)
 			return -NARGS(oparg)-2;
 		case MAKE_FUNCTION:
 			return -NARGS(oparg) - ((oparg >> 16) & 0xffff);
+		case MAKE_CLOSURE:
+			return -1 - NARGS(oparg) - ((oparg >> 16) & 0xffff);
 #undef NARGS
 		case BUILD_SLICE:
 			if (oparg == 3)
@@ -843,8 +845,6 @@ opcode_stack_effect(int opcode, int oparg)
 			else
 				return -1;
 
-		case MAKE_CLOSURE:
-			return -oparg;
 		case LOAD_CLOSURE:
 			return 1;
 		case LOAD_DEREF:
@@ -1367,8 +1367,12 @@ static int
 compiler_visit_annotations(struct compiler *c, arguments_ty args,
                            expr_ty returns)
 {
-	/* push arg annotations and a list of the argument names. return the #
-	   of items pushed. this is out-of-order wrt the source code. */
+	/* Push arg annotations and a list of the argument names. Return the #
+	   of items pushed. The expressions are evaluated out-of-order wrt the 
+	   source code. 
+	   
+	   More than 2^16-1 annotations is a SyntaxError. Returns -1 on error.
+	   */
 	static identifier return_str;
 	PyObject *names;
 	int len;
@@ -1399,6 +1403,12 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args,
 	}
 
 	len = PyList_GET_SIZE(names);
+	if (len > 65534) {
+		/* len must fit in 16 bits, and len is incremented below */
+		PyErr_SetString(PyExc_SyntaxError,
+				"too many annotations");
+		goto error;
+	}	
 	if (len) {
 		/* convert names to a tuple and place on stack */
 		PyObject *elt;
@@ -1449,6 +1459,9 @@ compiler_function(struct compiler *c, stmt_ty s)
 	if (args->defaults)
 		VISIT_SEQ(c, expr, args->defaults);
 	num_annotations = compiler_visit_annotations(c, args, returns);
+	if (num_annotations < 0)
+		return 0;
+	assert((num_annotations & 0xFFFF) == num_annotations);
 
 	if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s,
 				  s->lineno))

Python/peephole.c

@@ -261,10 +261,12 @@ markblocks(unsigned char *code, int len)
    The consts object should still be in list form to allow new constants 
    to be appended.
 
-   To keep the optimizer simple, it bails out (does nothing) for code
-   containing extended arguments or that has a length over 32,700.  That 
-   allows us to avoid overflow and sign issues.	 Likewise, it bails when
-   the lineno table has complex encoding for gaps >= 255.
+   To keep the optimizer simple, it bails out (does nothing) for code that
+   has a length over 32,700, and does not calculate extended arguments. 
+   That allows us to avoid overflow and sign issues. Likewise, it bails when
+   the lineno table has complex encoding for gaps >= 255. EXTENDED_ARG can
+   appear before MAKE_FUNCTION; in this case both opcodes are skipped.
+   EXTENDED_ARG preceding any other opcode causes the optimizer to bail.
 
    Optimizations are restricted to simple transformations occuring within a
    single basic block.	All transformations keep the code size the same or 
@@ -535,7 +537,11 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
 				break;
 
 			case EXTENDED_ARG:
+				if (codestr[i+3] != MAKE_FUNCTION)
 					goto exitUnchanged;
+				/* don't visit MAKE_FUNCTION as GETARG will be wrong */
+				i += 3;
+				break;
 
 				/* Replace RETURN LOAD_CONST None RETURN with just RETURN */
 				/* Remove unreachable JUMPs after RETURN */