Merged revisions 67066 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r67066 | benjamin.peterson | 2008-10-30 21:16:05 -0500 (Thu, 30 Oct 2008) | 5 lines

  make sure the parser flags and passed onto the compiler

  This fixes "from __future__ import unicode_literals" in an exec statment
  See #4225
........

Lib/test/test_future.py

@@ -106,6 +106,11 @@ class FutureTest(unittest.TestCase):
         support.unload("test.test_future5")
         from test import test_future5
 
+    def test_unicode_literals_exec(self):
+        scope = {}
+        exec("from __future__ import unicode_literals; x = ''", {}, scope)
+        self.assertTrue(isinstance(scope["x"], str))
+
 
 def test_main():
     support.run_unittest(FutureTest)

Lib/test/test_parser.py

@@ -25,6 +25,15 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
     def check_expr(self, s):
         self.roundtrip(parser.expr, s)
 
+    def test_flags_passed(self):
+        # The unicode literals flags has to be passed from the paser to AST
+        # generation.
+        suite = parser.suite("from __future__ import unicode_literals; x = ''")
+        code = suite.compile()
+        scope = {}
+        exec(code, {}, scope)
+        self.assertTrue(isinstance(scope["x"], str))
+
     def check_suite(self, s):
         self.roundtrip(parser.suite, s)
 

Modules/parsermodule.c

@@ -26,12 +26,20 @@
  */
 
 #include "Python.h"                     /* general Python API             */
+#include "Python-ast.h"                 /* mod_ty */
 #include "graminit.h"                   /* symbols defined in the grammar */
 #include "node.h"                       /* internal parser structure      */
 #include "errcode.h"                    /* error codes for PyNode_*()     */
 #include "token.h"                      /* token definitions              */
+#include "grammar.h"
+#include "parsetok.h"
                                         /* ISTERMINAL() / ISNONTERMINAL() */
-#include "compile.h"                    /* PyNode_Compile()               */
+#include "compile.h"
+#undef Yield
+#include "ast.h"
+#include "pyarena.h"
+
+extern grammar _PyParser_Grammar; /* From graminit.c */
 
 #ifdef lint
 #include <note.h>
@@ -156,6 +164,7 @@ typedef struct {
     PyObject_HEAD                       /* standard object header           */
     node* st_node;                      /* the node* returned by the parser */
     int   st_type;                      /* EXPR or SUITE ?                  */
+    PyCompilerFlags st_flags;           /* Parser and compiler flags        */
 } PyST_Object;
 
 
@@ -287,6 +296,7 @@ parser_newstobject(node *st, int type)
     if (o != 0) {
         o->st_node = st;
         o->st_type = type;
+        o->st_flags.cf_flags = 0;
     }
     else {
         PyNode_Free(st);
@@ -405,6 +415,8 @@ static PyObject*
 parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw)
 {
     PyObject*     res = 0;
+    PyArena*      arena;
+    mod_ty        mod;
     char*         str = "<syntax-tree>";
     int ok;
 
@@ -417,8 +429,16 @@ parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw)
         ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1],
                                          &str);
 
-    if (ok)
+    if (ok) {
-        res = (PyObject *)PyNode_Compile(self->st_node, str);
+        arena = PyArena_New();
+        if (arena) {
+           mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena);
+           if (mod) {
+               res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena);
+           }
+           PyArena_Free(arena);
+        }
+    }
 
     return (res);
 }
@@ -500,16 +520,25 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
 {
     char*     string = 0;
     PyObject* res    = 0;
+    int flags        = 0;
+    perrdetail err;
 
     static char *keywords[] = {"source", NULL};
 
     if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) {
-        node* n = PyParser_SimpleParseString(string,
+        node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL,
-                                             (type == PyST_EXPR)
+                                                       &_PyParser_Grammar,
-                                             ? eval_input : file_input);
+                                                      (type == PyST_EXPR)
+                                                      ? eval_input : file_input,
+                                                      &err, &flags);
 
-	if (n)
+	if (n) {
 	    res = parser_newstobject(n, type);
+            if (res)
+                ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK;
+        }
+        else
+            PyParser_SetError(&err);
     }
     return (res);
 }

Python/pythonrun.c

@@ -1688,16 +1688,19 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
 		       PyCompilerFlags *flags, PyArena *arena)
 {
 	mod_ty mod;
+	PyCompilerFlags localflags;
 	perrdetail err;
 	int iflags = PARSER_FLAGS(flags);
 
 	node *n = PyParser_ParseStringFlagsFilenameEx(s, filename,
 					&_PyParser_Grammar, start, &err,
 					&iflags);
+	if (flags == NULL) {
+		localflags.cf_flags = 0;
+		flags = &localflags;
+	}
 	if (n) {
-		if (flags) {
+		flags->cf_flags |= iflags & PyCF_MASK;
-			flags->cf_flags |= iflags & PyCF_MASK;
-		}
 		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;
@@ -1715,16 +1718,19 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
 		     PyArena *arena)
 {
 	mod_ty mod;
+	PyCompilerFlags localflags;
 	perrdetail err;
 	int iflags = PARSER_FLAGS(flags);
 
 	node *n = PyParser_ParseFileFlagsEx(fp, filename, enc,
 					  &_PyParser_Grammar,
 				start, ps1, ps2, &err, &iflags);
+	if (flags == NULL) {
+		localflags.cf_flags = 0;
+		flags = &localflags;
+	}
 	if (n) {
-		if (flags) {
+		flags->cf_flags |= iflags & PyCF_MASK;
-			flags->cf_flags |= iflags & PyCF_MASK;
-		}
 		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;