Bug #1191458: tracing over for loops now produces a line event

on each iteration.  I'm not positive this is the best way to handle
this.  I'm also not sure that there aren't other cases where
the lnotab is generated incorrectly.  It would be great if people
that use pdb or tracing could test heavily.

Also:
 * Remove dead/duplicated code that wasn't used/necessary
   because we already handled the docstring prior to entering the loop.
 * add some debugging code into the compiler (#if 0'd out).

Lib/test/test_trace.py

@@ -244,8 +244,8 @@ class TraceTestCase(unittest.TestCase):
         self.run_test(one_instr_line)
     def test_04_no_pop_blocks(self):
         self.run_test(no_pop_blocks)
-##    def test_05_no_pop_tops(self):
-##        self.run_test(no_pop_tops)
+    def test_05_no_pop_tops(self):
+        self.run_test(no_pop_tops)
     def test_06_call(self):
         self.run_test(call)
     def test_07_raise(self):

Misc/NEWS

@@ -12,6 +12,11 @@ What's New in Python 2.5 release candidate 1?
 Core and builtins
 -----------------
 
+- Bug #1191458: tracing over for loops now produces a line event
+  on each iteration.  Fixing this problem required changing the .pyc
+  magic number.  This means that .pyc files generated before 2.5c1
+  will be regenerated.
+
 
 Library
 -------
@@ -83,7 +88,7 @@ Core and builtins
 
 - Bug #1520864: unpacking singleton tuples in a 'for' loop (for x, in) works
   again.  Fixing this problem required changing the .pyc magic number.
-  This means that .pyc files generated before 2.5c1 will be regenerated.
+  This means that .pyc files generated before 2.5b3 will be regenerated.
 
 - Bug #1524317:  Compiling Python ``--without-threads`` failed.
   The Python core compiles again then, and, in a build without threads, the

Python/compile.c

@@ -143,7 +143,7 @@ struct compiler {
 	PyFutureFeatures *c_future; /* pointer to module's __future__ */
 	PyCompilerFlags *c_flags;
 
-	int c_interactive;
+	int c_interactive;	 /* true if in interactive mode */
 	int c_nestlevel;
 
 	struct compiler_unit *u; /* compiler state for current block */
@@ -1990,11 +1990,8 @@ compiler_function(struct compiler *c, stmt_ty s)
 	n = asdl_seq_LEN(s->v.FunctionDef.body);
 	/* if there was a docstring, we need to skip the first statement */
 	for (i = docstring; i < n; i++) {
-		stmt_ty s2 = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i);
-		if (i == 0 && s2->kind == Expr_kind &&
-		    s2->v.Expr.value->kind == Str_kind)
-			continue;
-		VISIT_IN_SCOPE(c, stmt, s2);
+		st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i);
+		VISIT_IN_SCOPE(c, stmt, st);
 	}
 	co = assemble(c, 1);
 	compiler_exit_scope(c);
@@ -2217,6 +2214,10 @@ compiler_for(struct compiler *c, stmt_ty s)
 	VISIT(c, expr, s->v.For.iter);
 	ADDOP(c, GET_ITER);
 	compiler_use_next_block(c, start);
+	/* XXX(nnorwitz): is there a better way to handle this?
+	   for loops are special, we want to be able to trace them
+	   each time around, so we need to set an extra line number. */
+	c->u->u_lineno_set = false;
 	ADDOP_JREL(c, FOR_ITER, cleanup);
 	VISIT(c, expr, s->v.For.target);
 	VISIT_SEQ(c, stmt, s->v.For.body);
@@ -4139,7 +4140,10 @@ assemble_lnotab(struct assembler *a, struct instr *i)
 	assert(d_bytecode >= 0);
 	assert(d_lineno >= 0);
 
-	if (d_lineno == 0)
+	/* XXX(nnorwitz): is there a better way to handle this?
+	   for loops are special, we want to be able to trace them
+	   each time around, so we need to set an extra line number. */
+	if (d_lineno == 0 && i->i_opcode != FOR_ITER)
 		return 1;
 
 	if (d_bytecode > 255) {
@@ -4444,6 +4448,41 @@ makecode(struct compiler *c, struct assembler *a)
 	return co;
 }
 
+
+/* For debugging purposes only */
+#if 0
+static void
+dump_instr(const struct instr *i)
+{
+	const char *jrel = i->i_jrel ? "jrel " : "";
+	const char *jabs = i->i_jabs ? "jabs " : "";
+	char arg[128];
+
+	*arg = '\0';
+	if (i->i_hasarg)
+		sprintf(arg, "arg: %d ", i->i_oparg);
+
+	fprintf(stderr, "line: %d, opcode: %d %s%s%s\n", 
+			i->i_lineno, i->i_opcode, arg, jabs, jrel);
+}
+
+static void
+dump_basicblock(const basicblock *b)
+{
+	const char *seen = b->b_seen ? "seen " : "";
+	const char *b_return = b->b_return ? "return " : "";
+	fprintf(stderr, "used: %d, depth: %d, offset: %d %s%s\n",
+		b->b_iused, b->b_startdepth, b->b_offset, seen, b_return);
+	if (b->b_instr) {
+		int i;
+		for (i = 0; i < b->b_iused; i++) {
+			fprintf(stderr, "  [%02d] ", i);
+			dump_instr(b->b_instr + i);
+		}
+	}
+}
+#endif
+
 static PyCodeObject *
 assemble(struct compiler *c, int addNone)
 {

Python/import.c

@@ -62,9 +62,10 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
        Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
        Python 2.5b3: 62101 (fix wrong code: for x, in ...)
        Python 2.5b3: 62111 (fix wrong code: x += yield)
+       Python 2.5c1: 62121 (fix wrong lnotab with for loops)
 .
 */
-#define MAGIC (62111 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (62121 | ((long)'\r'<<16) | ((long)'\n'<<24))
 
 /* Magic word as global; note that _PyImport_Init() can change the
    value of this global to accommodate for alterations of how the