From dcee09d9204957ace381b1c3507259f718aa4907 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 31 Oct 2008 02:16:05 +0000 Subject: [PATCH] make sure the parser flags and passed onto the compiler This fixes "from __future__ import unicode_literals" in an exec statment See #4225 --- Lib/test/test_future.py | 5 +++++ Lib/test/test_parser.py | 9 +++++++++ Misc/NEWS | 3 +++ Modules/parsermodule.c | 43 ++++++++++++++++++++++++++++++++++------- Python/pythonrun.c | 18 +++++++++++------ 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index 1432e749006..81d0a3ed2a0 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -106,6 +106,11 @@ class FutureTest(unittest.TestCase): test_support.unload("test.test_future5") from test import test_future5 + def test_unicode_literals_exec(self): + scope = {} + exec "from __future__ import unicode_literals; x = ''" in scope + self.assertTrue(isinstance(scope["x"], unicode)) + def test_main(): test_support.run_unittest(FutureTest) diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index ba77bb806f3..9ecca51a360 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -25,6 +25,15 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase): def check_expr(self, s): self.roundtrip(parser.expr, s) + def test_flags_passed(self): + # The unicode literals flags has to be passed from the paser to AST + # generation. + suite = parser.suite("from __future__ import unicode_literals; x = ''") + code = suite.compile() + scope = {} + exec code in scope + self.assertTrue(isinstance(scope["x"], unicode)) + def check_suite(self, s): self.roundtrip(parser.suite, s) diff --git a/Misc/NEWS b/Misc/NEWS index 3017d4b52d5..234a15d1cc0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ What's New in Python 2.7 alpha 1 Core and Builtins ----------------- +- Issue #4225: ``from __future__ import unicode_literals`` didn't work in an + exec statement. + - Issue #4176: Fixed a crash when pickling an object which ``__reduce__`` method does not return iterators for the 4th and 5th items. diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 915bbdc7b2e..423a0b35006 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -26,12 +26,20 @@ */ #include "Python.h" /* general Python API */ +#include "Python-ast.h" /* mod_ty */ #include "graminit.h" /* symbols defined in the grammar */ #include "node.h" /* internal parser structure */ #include "errcode.h" /* error codes for PyNode_*() */ #include "token.h" /* token definitions */ +#include "grammar.h" +#include "parsetok.h" /* ISTERMINAL() / ISNONTERMINAL() */ -#include "compile.h" /* PyNode_Compile() */ +#include "compile.h" +#undef Yield +#include "ast.h" +#include "pyarena.h" + +extern grammar _PyParser_Grammar; /* From graminit.c */ #ifdef lint #include @@ -156,6 +164,7 @@ typedef struct { PyObject_HEAD /* standard object header */ node* st_node; /* the node* returned by the parser */ int st_type; /* EXPR or SUITE ? */ + PyCompilerFlags st_flags; /* Parser and compiler flags */ } PyST_Object; @@ -260,6 +269,7 @@ parser_newstobject(node *st, int type) if (o != 0) { o->st_node = st; o->st_type = type; + o->st_flags.cf_flags = 0; } else { PyNode_Free(st); @@ -394,6 +404,8 @@ static PyObject* parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) { PyObject* res = 0; + PyArena* arena; + mod_ty mod; char* str = ""; int ok; @@ -406,8 +418,16 @@ parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) ok = PyArg_ParseTupleAndKeywords(args, kw, "|s:compile", &keywords[1], &str); - if (ok) - res = (PyObject *)PyNode_Compile(self->st_node, str); + if (ok) { + arena = PyArena_New(); + if (arena) { + mod = PyAST_FromNode(self->st_node, &(self->st_flags), str, arena); + if (mod) { + res = (PyObject *)PyAST_Compile(mod, str, &(self->st_flags), arena); + } + PyArena_Free(arena); + } + } return (res); } @@ -523,16 +543,25 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) { char* string = 0; PyObject* res = 0; + int flags = 0; + perrdetail err; static char *keywords[] = {"source", NULL}; if (PyArg_ParseTupleAndKeywords(args, kw, argspec, keywords, &string)) { - node* n = PyParser_SimpleParseString(string, - (type == PyST_EXPR) - ? eval_input : file_input); + node* n = PyParser_ParseStringFlagsFilenameEx(string, NULL, + &_PyParser_Grammar, + (type == PyST_EXPR) + ? eval_input : file_input, + &err, &flags); - if (n) + if (n) { res = parser_newstobject(n, type); + if (res) + ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; + } + else + PyParser_SetError(&err); } return (res); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index bdd9bd7d881..4ff70d85430 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1417,16 +1417,19 @@ PyParser_ASTFromString(const char *s, const char *filename, int start, PyCompilerFlags *flags, PyArena *arena) { mod_ty mod; + PyCompilerFlags localflags; perrdetail err; int iflags = PARSER_FLAGS(flags); node *n = PyParser_ParseStringFlagsFilenameEx(s, filename, &_PyParser_Grammar, start, &err, &iflags); + if (flags == NULL) { + localflags.cf_flags = 0; + flags = &localflags; + } if (n) { - if (flags) { - flags->cf_flags |= iflags & PyCF_MASK; - } + flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); return mod; @@ -1443,15 +1446,18 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, PyArena *arena) { mod_ty mod; + PyCompilerFlags localflags; perrdetail err; int iflags = PARSER_FLAGS(flags); node *n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, start, ps1, ps2, &err, &iflags); + if (flags == NULL) { + localflags.cf_flags = 0; + flags = &localflags; + } if (n) { - if (flags) { - flags->cf_flags |= iflags & PyCF_MASK; - } + flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); return mod;