- Finally fixed the bug in compile() and exec where a string ending

with an indented code block but no newline would raise SyntaxError.
  This would have been a four-line change in parsetok.c...  Except
  codeop.py depends on this behavior, so a compilation flag had to be
  invented that causes the tokenizer to revert to the old behavior;
  this required extra changes to 2 .h files, 2 .c files, and 2 .py
  files.  (Fixes SF bug #501622.)

Include/parsetok.h

@@ -21,6 +21,8 @@ typedef struct {
 #define PyPARSE_YIELD_IS_KEYWORD	0x0001
 #endif
 
+#define PyPARSE_DONT_IMPLY_DEDENT	0x0002
+
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
 PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,

Include/pythonrun.h

@@ -10,6 +10,7 @@ extern "C" {
 #define PyCF_MASK (CO_FUTURE_DIVISION)
 #define PyCF_MASK_OBSOLETE (CO_GENERATOR_ALLOWED | CO_NESTED)
 #define PyCF_SOURCE_IS_UTF8  0x0100
+#define PyCF_DONT_IMPLY_DEDENT 0x0200
 
 typedef struct {
 	int cf_flags;  /* bitmask of CO_xxx flags relevant to future */

Lib/code.py

@@ -303,4 +303,5 @@ def interact(banner=None, readfunc=None, local=None):
 
 
 if __name__ == '__main__':
-    interact()
+    import pdb
+    pdb.run("interact()\n")

Lib/codeop.py

@@ -63,6 +63,8 @@ _features = [getattr(__future__, fname)
 
 __all__ = ["compile_command", "Compile", "CommandCompiler"]
 
+PyCF_DONT_IMPLY_DEDENT = 0x200          # Matches pythonrun.h
+
 def _maybe_compile(compiler, source, filename, symbol):
     # Check for source consisting of only blank lines and comments
     for line in source.split("\n"):
@@ -103,6 +105,9 @@ def _maybe_compile(compiler, source, filename, symbol):
     if not code1 and e1 == e2:
         raise SyntaxError, err1
 
+def _compile(source, filename, symbol):
+    return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
+
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
 
@@ -121,7 +126,7 @@ def compile_command(source, filename="<input>", symbol="single"):
       syntax error (OverflowError and ValueError can be produced by
       malformed literals).
     """
-    return _maybe_compile(compile, source, filename, symbol)
+    return _maybe_compile(_compile, source, filename, symbol)
 
 class Compile:
     """Instances of this class behave much like the built-in compile
@@ -129,7 +134,7 @@ class Compile:
     statement, it "remembers" and compiles all subsequent program texts
     with the statement in force."""
     def __init__(self):
-        self.flags = 0
+        self.flags = PyCF_DONT_IMPLY_DEDENT
 
     def __call__(self, source, filename, symbol):
         codeob = compile(source, filename, symbol, self.flags, 1)

Lib/test/test_codeop.py

@@ -5,13 +5,13 @@
 import unittest
 from test.test_support import run_unittest
 
-from codeop import compile_command
+from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 
 class CodeopTests(unittest.TestCase):
 
     def assertValid(self, str, symbol='single'):
         '''succeed iff str is a valid piece of code'''
-        expected = compile(str, "<input>", symbol)
+        expected = compile(str, "<input>", symbol, PyCF_DONT_IMPLY_DEDENT)
         self.assertEquals( compile_command(str, "<input>", symbol), expected)
 
 
@@ -42,7 +42,8 @@ class CodeopTests(unittest.TestCase):
 
         # special case
         self.assertEquals(compile_command(""),
-                          compile("pass", "<input>", 'single'))
+                          compile("pass", "<input>", 'single',
+                                  PyCF_DONT_IMPLY_DEDENT))
 
         av("3**3","eval")
         av("(lambda z: \n z**3)","eval")

Lib/test/test_compile.py

@@ -89,6 +89,15 @@ expect_error("2.0e+")
 expect_error("1e-")
 expect_error("3-4e/21")
 
+if verbose:
+    print "testing compile() of indented block w/o trailing newline"
+
+s = """
+if 1:
+    if 2:
+        pass"""
+compile(s, "<string>", "exec")
+
 
 if verbose:
     print "testing literals with leading zeroes"

Misc/NEWS

@@ -12,6 +12,14 @@ What's New in Python 2.3 alpha 2?
 Core and builtins
 -----------------
 
+- Finally fixed the bug in compile() and exec where a string ending
+  with an indented code block but no newline would raise SyntaxError.
+  This would have been a four-line change in parsetok.c...  Except
+  codeop.py depends on this behavior, so a compilation flag had to be
+  invented that causes the tokenizer to revert to the old behavior;
+  this required extra changes to 2 .h files, 2 .c files, and 2 .py
+  files.
+
 - If a new-style class defines neither __new__ nor __init__, its
   constructor would ignore all arguments.  This is changed now: the
   constructor refuses arguments in this case.  This might break code

Parser/parsetok.c

@@ -130,6 +130,15 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 		if (type == ENDMARKER && started) {
 			type = NEWLINE; /* Add an extra newline */
 			started = 0;
+			/* Add the right number of dedent tokens,
+			   except if a certain flag is given --
+			   codeop.py uses this. */
+			if (tok->indent &&
+			    !(flags & PyPARSE_DONT_IMPLY_DEDENT))
+			{
+				tok->pendin = -tok->indent;
+				tok->indent = 0;
+			}
 		}
 		else
 			started = 1;

Python/bltinmodule.c

@@ -380,7 +380,9 @@ builtin_compile(PyObject *self, PyObject *args)
 		return NULL;
 	}
 
-	if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE)) {
+	if (supplied_flags &
+	    ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT))
+	{
 		PyErr_SetString(PyExc_ValueError,
 				"compile(): unrecognised flags");
 		return NULL;

Python/pythonrun.c

@@ -548,13 +548,9 @@ PyRun_InteractiveOne(FILE *fp, const char *filename)
 }
 
 /* compute parser flags based on compiler flags */
-#if 0 /* future keyword */
 #define PARSER_FLAGS(flags) \
-	(((flags) && (flags)->cf_flags & CO_GENERATOR_ALLOWED) ? \
+	(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
-		PyPARSE_YIELD_IS_KEYWORD : 0)
+		PyPARSE_DONT_IMPLY_DEDENT : 0)
-#else
-#define PARSER_FLAGS(flags) 0
-#endif
 
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)