traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Hide list comp variables and support set comprehensions

This commit is contained in:
Nick Coghlan 2007-04-15 12:05:43 +00:00
parent 6ef6306dd6
commit 650f0d06d3
29 changed files with 2006 additions and 1323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Doc/lib/libdis.tex
View File

@ -363,6 +363,10 @@ is the address to jump to (which should be a \code{FOR_ITER}
instruction).
\end{opcodedesc}
\begin{opcodedesc}{SET_ADD}{}
Calls \code{set.add(TOS1, TOS)}. Used to implement set comprehensions.
\end{opcodedesc}
\begin{opcodedesc}{LIST_APPEND}{}
Calls \code{list.append(TOS1, TOS)}. Used to implement list comprehensions.
\end{opcodedesc}

37
Grammar/Grammar
View File

@ -82,16 +82,10 @@ with_var: 'as' expr
except_clause: 'except' [test ['as' NAME]]
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
# Backward compatibility cruft to support:
# [ x for x in lambda: True, lambda: False if x() ]
# even while also allowing:
# lambda x: 5 if x else 2
# (But not a mix of the two)
testlist_safe: old_test [(',' old_test)+ [',']]
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
@ -105,33 +99,28 @@ arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
NAME | NUMBER | STRING+ | '...')
listmaker: test ( list_for | (',' test)* [','] )
testlist_gexp: test ( gen_for | (',' test)* [','] )
lambdef: 'lambda' [varargslist] ':' test
testlist_comp: test ( comp_for | (',' test)* [','] )
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: expr (',' expr)* [',']
testlist: test (',' test)* [',']
dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
dictorsetmaker: ( (test ':' test (',' test ':' test)* [',']) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
argument: test [gen_for] | test '=' test # Really [keyword '='] test
argument: test [comp_for] | test '=' test # Really [keyword '='] test
list_iter: list_for | list_if
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_if: 'if' old_test [list_iter]
gen_iter: gen_for | gen_if
gen_for: 'for' exprlist 'in' or_test [gen_iter]
gen_if: 'if' old_test [gen_iter]
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' test_nocond [comp_iter]
testlist1: test (',' test)*

16
Include/Python-ast.h
View File

@ -183,10 +183,10 @@ struct _stmt {
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
GeneratorExp_kind=9, Yield_kind=10, Compare_kind=11,
Call_kind=12, Num_kind=13, Str_kind=14, Bytes_kind=15,
Ellipsis_kind=16, Attribute_kind=17, Subscript_kind=18,
Name_kind=19, List_kind=20, Tuple_kind=21};
SetComp_kind=9, GeneratorExp_kind=10, Yield_kind=11,
Compare_kind=12, Call_kind=13, Num_kind=14, Str_kind=15,
Bytes_kind=16, Ellipsis_kind=17, Attribute_kind=18,
Subscript_kind=19, Name_kind=20, List_kind=21, Tuple_kind=22};
struct _expr {
enum _expr_kind kind;
union {
@ -231,6 +231,11 @@ struct _expr {
asdl_seq *generators;
} ListComp;
struct {
expr_ty elt;
asdl_seq *generators;
} SetComp;
struct {
expr_ty elt;
asdl_seq *generators;
@ -465,6 +470,9 @@ expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);

47
Include/graminit.h
View File

@ -46,10 +46,10 @@
#define with_var 301
#define except_clause 302
#define suite 303
#define testlist_safe 304
#define old_test 305
#define old_lambdef 306
#define test 307
#define test 304
#define test_nocond 305
#define lambdef 306
#define lambdef_nocond 307
#define or_test 308
#define and_test 309
#define not_test 310
@ -64,25 +64,20 @@
#define factor 319
#define power 320
#define atom 321
#define listmaker 322
#define testlist_gexp 323
#define lambdef 324
#define trailer 325
#define subscriptlist 326
#define subscript 327
#define sliceop 328
#define exprlist 329
#define testlist 330
#define dictsetmaker 331
#define classdef 332
#define arglist 333
#define argument 334
#define list_iter 335
#define list_for 336
#define list_if 337
#define gen_iter 338
#define gen_for 339
#define gen_if 340
#define testlist1 341
#define encoding_decl 342
#define yield_expr 343
#define testlist_comp 322
#define trailer 323
#define subscriptlist 324
#define subscript 325
#define sliceop 326
#define exprlist 327
#define testlist 328
#define dictorsetmaker 329
#define classdef 330
#define arglist 331
#define argument 332
#define comp_iter 333
#define comp_for 334
#define comp_if 335
#define testlist1 336
#define encoding_decl 337
#define yield_expr 338

1
Include/opcode.h
View File

@ -21,6 +21,7 @@ extern "C" {
#define UNARY_INVERT 15
#define SET_ADD 17
#define LIST_APPEND 18
#define BINARY_POWER 19

32
Include/symtable.h
View File

@ -5,31 +5,35 @@
extern "C" {
#endif
/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal
* names.
*/
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
_Py_block_ty;
struct _symtable_entry;
struct symtable {
const char *st_filename; /* name of file being compiled */
const char *st_filename; /* name of file being compiled */
struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* module entry */
PyObject *st_symbols; /* dictionary of symbol table entries */
PyObject *st_stack; /* stack of namespace info */
PyObject *st_global; /* borrowed ref to MODULE in st_symbols */
int st_nblocks; /* number of blocks */
PyObject *st_private; /* name of current class or NULL */
int st_tmpname; /* temporary name counter */
PyFutureFeatures *st_future; /* module's future features */
struct _symtable_entry *st_top; /* symbol table entry for module */
PyObject *st_blocks; /* dict: map AST node addresses
* to symbol table entries */
PyObject *st_stack; /* list: stack of namespace info */
PyObject *st_global; /* borrowed ref to st_top->st_symbols */
int st_nblocks; /* number of blocks used */
PyObject *st_private; /* name of current class or NULL */
PyFutureFeatures *st_future; /* module's future features */
};
typedef struct _symtable_entry {
PyObject_HEAD
PyObject *ste_id; /* int: key in st_symbols */
PyObject *ste_symbols; /* dict: name to flags */
PyObject *ste_name; /* string: name of block */
PyObject *ste_id; /* int: key in ste_table->st_blocks */
PyObject *ste_symbols; /* dict: variable names to flags */
PyObject *ste_name; /* string: name of current block */
PyObject *ste_varnames; /* list of variable names */
PyObject *ste_children; /* list of child ids */
PyObject *ste_children; /* list of child blocks */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
unsigned ste_nested : 1; /* true if block is nested */
@ -80,7 +84,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
table. GLOBAL is returned from PyST_GetScope() for either of them.
It is stored in ste_symbols at bits 12-15.
*/
#define SCOPE_OFF 11
#define SCOPE_OFFSET 11
#define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL)
#define LOCAL 1

4
Lib/compiler/transformer.py
View File

@ -559,7 +559,7 @@ class Transformer:
testlist1 = testlist
exprlist = testlist
def testlist_gexp(self, nodelist):
def testlist_comp(self, nodelist):
if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for:
test = self.com_node(nodelist[0])
return self.com_generator_expression(test, nodelist[1])
@ -1027,7 +1027,7 @@ class Transformer:
# loop to avoid trivial recursion
while 1:
t = node[0]
if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp):
if t in (symbol.exprlist, symbol.testlist, symbol.testlist_comp):
if len(node) > 2:
return self.com_assign_tuple(node, assigning)
node = node[1]

2
Lib/decimal.py
View File

@ -2282,10 +2282,8 @@ class Context(object):
_ignored_flags = []
if not isinstance(flags, dict):
flags = dict([(s,s in flags) for s in _signals])
del s
if traps is not None and not isinstance(traps, dict):
traps = dict([(s,s in traps) for s in _signals])
del s
for name, val in locals().items():
if val is None:
setattr(self, name, _copy.copy(getattr(DefaultContext, name)))

1
Lib/opcode.py
View File

@ -57,6 +57,7 @@ def_op('UNARY_NOT', 12)
def_op('UNARY_INVERT', 15)
def_op('SET_ADD', 17)
def_op('LIST_APPEND', 18)
def_op('BINARY_POWER', 19)
def_op('BINARY_MULTIPLY', 20)

1
Lib/pickle.py
View File

@ -163,7 +163,6 @@ _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
del x
# Pickling machinery

4
Lib/test/test_datetime.py
View File

@ -20,9 +20,9 @@ from datetime import time
from datetime import date, datetime
pickle_choices = [(pickler, unpickler, proto)
for pickler in pickle, cPickle
for pickler in (pickle, cPickle)
if pickler is not None
for unpickler in pickle, cPickle
for unpickler in (pickle, cPickle)
if unpickler is not None
for proto in range(3)]
if cPickle is None:

8
Lib/test/test_dis.py
View File

@ -129,8 +129,12 @@ class DisTests(unittest.TestCase):
def test_bug_1333982(self):
# This one is checking bytecodes generated for an `assert` statement,
# so fails if the tests are run with -O. Skip this test then.
if __debug__:
self.do_disassembly_test(bug1333982, dis_bug1333982)
pass # Test has been disabled due to change in the way
# list comps are handled. The byte code now includes
# a memory address and a file location, so they change from
# run to run.
# if __debug__:
# self.do_disassembly_test(bug1333982, dis_bug1333982)
def test_big_linenos(self):
def func(count):

3
Lib/test/test_grammar.py
View File

@ -843,7 +843,8 @@ class GrammarTests(unittest.TestCase):
print(x)
return ret
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
# the next line is not allowed anymore
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)

444
Lib/test/test_listcomps.py Normal file
View File

@ -0,0 +1,444 @@
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests borrowed from or inspired by test_genexps.py ############
Test simple loop with conditional
>>> sum([i*i for i in range(100) if i&1 == 1])
166650
Test simple nesting
>>> [(i,j) for i in range(3) for j in range(4)]
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> [(i,j) for i in range(4) for j in range(i)]
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum([i*i for i in range(100)])
328350
>>> i
20
Verify that syntax error's are raised for listcomps used as lvalues
>>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
########### Tests borrowed from or inspired by test_generators.py ############
Make a nested list comprehension that acts like range()
>>> def frange(n):
... return [i for i in xrange(n)]
>>> frange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: [i for i in xrange(n)]
>>> lrange(10)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in [i for i in xrange(n)]:
... yield x
>>> list(grange(5))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> [None for i in xrange(10)]
[None, None, None, None, None, None, None, None, None, None]
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = [(lambda i=i: i) for i in range(5)]
>>> [x() for x in items]
[0, 1, 2, 3, 4]
Same again, only this time as a closure variable
>>> items = [(lambda: i) for i in range(5)]
>>> [x() for x in items]
[4, 4, 4, 4, 4]
Another way to test that the iteration variable is local to the list comp
>>> items = [(lambda: i) for i in range(5)]
>>> i = 20
>>> [x() for x in items]
[4, 4, 4, 4, 4]
And confirm that a closure can jump over the list comp scope
>>> items = [(lambda: y) for i in range(5)]
>>> y = 2
>>> [x() for x in items]
[2, 2, 2, 2, 2]
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = [(lambda i=i: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[0, 1, 2, 3, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: i) for i in range(5)]
... i = 20
... return [x() for x in items]
>>> test_func()
[4, 4, 4, 4, 4]
>>> def test_func():
... items = [(lambda: y) for i in range(5)]
... y = 2
... return [x() for x in items]
>>> test_func()
[2, 2, 2, 2, 2]
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)

453
Lib/test/test_setcomps.py Normal file
View File

@ -0,0 +1,453 @@
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)
doctests = """
########### Tests mostly copied from test_listcomps.py ############
Test simple loop with conditional
>>> sum({i*i for i in range(100) if i&1 == 1})
166650
Test simple case
>>> {2*y + x + 1 for x in (0,) for y in (1,)}
{3}
Test simple nesting
>>> list(sorted({(i,j) for i in range(3) for j in range(4)}))
[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)]
Test nesting with the inner expression dependent on the outer
>>> list(sorted({(i,j) for i in range(4) for j in range(i)}))
[(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)]
Make sure the induction variable is not exposed
>>> i = 20
>>> sum({i*i for i in range(100)})
328350
>>> i
20
Verify that syntax error's are raised for setcomps used as lvalues
>>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
>>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
SyntaxError: ...
Make a nested set comprehension that acts like set(xrange())
>>> def srange(n):
... return {i for i in xrange(n)}
>>> list(sorted(srange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Same again, only as a lambda expression instead of a function definition
>>> lrange = lambda n: {i for i in xrange(n)}
>>> list(sorted(lrange(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Generators can call other generators:
>>> def grange(n):
... for x in {i for i in xrange(n)}:
... yield x
>>> list(sorted(grange(5)))
[0, 1, 2, 3, 4]
Make sure that None is a valid return value
>>> {None for i in xrange(10)}
{None}
########### Tests for various scoping corner cases ############
Return lambdas that use the iteration variable as a default argument
>>> items = {(lambda i=i: i) for i in range(5)}
>>> {x() for x in items} == set(range(5))
True
Same again, only this time as a closure variable
>>> items = {(lambda: i) for i in range(5)}
>>> {x() for x in items}
{4}
Another way to test that the iteration variable is local to the list comp
>>> items = {(lambda: i) for i in range(5)}
>>> i = 20
>>> {x() for x in items}
{4}
And confirm that a closure can jump over the list comp scope
>>> items = {(lambda: y) for i in range(5)}
>>> y = 2
>>> {x() for x in items}
{2}
We also repeat each of the above scoping tests inside a function
>>> def test_func():
... items = {(lambda i=i: i) for i in range(5)}
... return {x() for x in items}
>>> test_func() == set(range(5))
True
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: i) for i in range(5)}
... i = 20
... return {x() for x in items}
>>> test_func()
{4}
>>> def test_func():
... items = {(lambda: y) for i in range(5)}
... y = 2
... return {x() for x in items}
>>> test_func()
{2}
"""
__test__ = {'doctests' : doctests}
def test_main(verbose=None):
import sys
from test import test_support
from test import test_listcomps
test_support.run_doctest(test_listcomps, verbose)
# verify reference counting
if verbose and hasattr(sys, "gettotalrefcount"):
import gc
counts = [None] * 5
for i in xrange(len(counts)):
test_support.run_doctest(test_genexps, verbose)
gc.collect()
counts[i] = sys.gettotalrefcount()
print(counts)
if __name__ == "__main__":
test_main(verbose=True)

4
Lib/test/test_syntax.py
View File

@ -5,7 +5,7 @@ Here's an example of the sort of thing that is tested.
>>> def f(x):
... global x
Traceback (most recent call last):
SyntaxError: name 'x' is local and global
SyntaxError: name 'x' is parameter and global
The tests are all raise SyntaxErrors. They were created by checking
each C call that raises SyntaxError. There are several modules that
@ -373,7 +373,7 @@ Misuse of the nonlocal statement can lead to a few unique syntax errors.
... nonlocal x
Traceback (most recent call last):
...
SyntaxError: name 'x' is local and nonlocal
SyntaxError: name 'x' is parameter and nonlocal
>>> def f():
... global x

1
Lib/test/test_univnewlines.py
View File

@ -28,7 +28,6 @@ DATA_CRLF = "\r\n".join(DATA_TEMPLATE) + "\r\n"
# before end-of-file.
DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r"
DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE]
del x
class TestGenericUnivNewlines(unittest.TestCase):
# use a class variable DATA to define the data to write to the file

1
Lib/tokenize.py
View File

@ -32,7 +32,6 @@ from token import *
import token
__all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize",
"generate_tokens", "NL", "untokenize"]
del x
del token
COMMENT = N_TOKENS

3
Misc/NEWS
View File

@ -28,6 +28,9 @@ TO DO
Core and Builtins
-----------------
- Patch #1660500: hide iteration variable in list comps, add set comps
and use common code to handle compilation of iterative expressions
- By default, != returns the opposite of ==, unless the latter returns
NotImplemented.

205
Modules/parsermodule.c
View File

@ -858,11 +858,10 @@ VALIDATER(vfpdef); VALIDATER(vfplist);
VALIDATER(stmt); VALIDATER(simple_stmt);
VALIDATER(expr_stmt); VALIDATER(power);
VALIDATER(del_stmt);
VALIDATER(return_stmt); VALIDATER(list_iter);
VALIDATER(raise_stmt); VALIDATER(import_stmt);
VALIDATER(import_name); VALIDATER(import_from);
VALIDATER(global_stmt); VALIDATER(list_if);
VALIDATER(assert_stmt); VALIDATER(list_for);
VALIDATER(return_stmt); VALIDATER(raise_stmt);
VALIDATER(import_stmt); VALIDATER(import_stmt);
VALIDATER(import_name); VALIDATER(yield_stmt);
VALIDATER(global_stmt); VALIDATER(assert_stmt);
VALIDATER(compound_stmt); VALIDATER(vname);
VALIDATER(while); VALIDATER(for);
VALIDATER(try); VALIDATER(except_clause);
@ -875,14 +874,13 @@ VALIDATER(term); VALIDATER(factor);
VALIDATER(atom); VALIDATER(lambdef);
VALIDATER(trailer); VALIDATER(subscript);
VALIDATER(subscriptlist); VALIDATER(sliceop);
VALIDATER(exprlist); VALIDATER(dictsetmaker);
VALIDATER(exprlist); VALIDATER(dictorsetmaker);
VALIDATER(arglist); VALIDATER(argument);
VALIDATER(listmaker); VALIDATER(yield_stmt);
VALIDATER(testlist1); VALIDATER(gen_for);
VALIDATER(gen_iter); VALIDATER(gen_if);
VALIDATER(testlist_gexp); VALIDATER(yield_expr);
VALIDATER(yield_or_testlist); VALIDATER(or_test);
VALIDATER(old_test); VALIDATER(old_lambdef);
VALIDATER(testlist1); VALIDATER(comp_for);
VALIDATER(comp_iter); VALIDATER(comp_if);
VALIDATER(testlist_comp); VALIDATER(yield_expr);
VALIDATER(yield_or_testlist); VALIDATER(or_test);
VALIDATER(test_nocond); VALIDATER(lambdef_nocond);
#undef VALIDATER
@ -1112,14 +1110,6 @@ validate_testlist1(node *tree)
}
static int
validate_testlist_safe(node *tree)
{
return (validate_repeating_list(tree, testlist_safe,
validate_old_test, "testlist_safe"));
}
/* validate either vname or tname.
* vname: NAME
* tname: NAME [':' test]
@ -1330,70 +1320,33 @@ validate_varargslist(node *tree)
}
/* list_iter: list_for | list_if
/* comp_iter: comp_for | comp_if
*/
static int
validate_list_iter(node *tree)
validate_comp_iter(node *tree)
{
int res = (validate_ntype(tree, list_iter)
&& validate_numnodes(tree, 1, "list_iter"));
if (res && TYPE(CHILD(tree, 0)) == list_for)
res = validate_list_for(CHILD(tree, 0));
int res = (validate_ntype(tree, comp_iter)
&& validate_numnodes(tree, 1, "comp_iter"));
if (res && TYPE(CHILD(tree, 0)) == comp_for)
res = validate_comp_for(CHILD(tree, 0));
else
res = validate_list_if(CHILD(tree, 0));
res = validate_comp_if(CHILD(tree, 0));
return res;
}
/* gen_iter: gen_for | gen_if
/* comp_for: 'for' exprlist 'in' test [comp_iter]
*/
static int
validate_gen_iter(node *tree)
{
int res = (validate_ntype(tree, gen_iter)
&& validate_numnodes(tree, 1, "gen_iter"));
if (res && TYPE(CHILD(tree, 0)) == gen_for)
res = validate_gen_for(CHILD(tree, 0));
else
res = validate_gen_if(CHILD(tree, 0));
return res;
}
/* list_for: 'for' exprlist 'in' testlist [list_iter]
*/
static int
validate_list_for(node *tree)
validate_comp_for(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 5)
res = validate_list_iter(CHILD(tree, 4));
res = validate_comp_iter(CHILD(tree, 4));
else
res = validate_numnodes(tree, 4, "list_for");
if (res)
res = (validate_name(CHILD(tree, 0), "for")
&& validate_exprlist(CHILD(tree, 1))
&& validate_name(CHILD(tree, 2), "in")
&& validate_testlist_safe(CHILD(tree, 3)));
return res;
}
/* gen_for: 'for' exprlist 'in' test [gen_iter]
*/
static int
validate_gen_for(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 5)
res = validate_gen_iter(CHILD(tree, 4));
else
res = validate_numnodes(tree, 4, "gen_for");
res = validate_numnodes(tree, 4, "comp_for");
if (res)
res = (validate_name(CHILD(tree, 0), "for")
@ -1404,45 +1357,26 @@ validate_gen_for(node *tree)
return res;
}
/* list_if: 'if' old_test [list_iter]
/* comp_if: 'if' test_nocond [comp_iter]
*/
static int
validate_list_if(node *tree)
validate_comp_if(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 3)
res = validate_list_iter(CHILD(tree, 2));
res = validate_comp_iter(CHILD(tree, 2));
else
res = validate_numnodes(tree, 2, "list_if");
res = validate_numnodes(tree, 2, "comp_if");
if (res)
res = (validate_name(CHILD(tree, 0), "if")
&& validate_old_test(CHILD(tree, 1)));
&& validate_test_nocond(CHILD(tree, 1)));
return res;
}
/* gen_if: 'if' old_test [gen_iter]
*/
static int
validate_gen_if(node *tree)
{
int nch = NCH(tree);
int res;
if (nch == 3)
res = validate_gen_iter(CHILD(tree, 2));
else
res = validate_numnodes(tree, 2, "gen_if");
if (res)
res = (validate_name(CHILD(tree, 0), "if")
&& validate_old_test(CHILD(tree, 1)));
return res;
}
/* validate_vfpdef()
*
@ -2089,13 +2023,13 @@ validate_test(node *tree)
}
static int
validate_old_test(node *tree)
validate_test_nocond(node *tree)
{
int nch = NCH(tree);
int res = validate_ntype(tree, old_test) && (nch == 1);
int res = validate_ntype(tree, test_nocond) && (nch == 1);
if (res && (TYPE(CHILD(tree, 0)) == old_lambdef))
res = (validate_old_lambdef(CHILD(tree, 0)));
if (res && (TYPE(CHILD(tree, 0)) == lambdef_nocond))
res = (validate_lambdef_nocond(CHILD(tree, 0)));
else if (res) {
res = (validate_or_test(CHILD(tree, 0)));
}
@ -2393,14 +2327,14 @@ validate_atom(node *tree)
if (TYPE(CHILD(tree, 1))==yield_expr)
res = validate_yield_expr(CHILD(tree, 1));
else
res = validate_testlist_gexp(CHILD(tree, 1));
res = validate_testlist_comp(CHILD(tree, 1));
}
break;
case LSQB:
if (nch == 2)
res = validate_ntype(CHILD(tree, 1), RSQB);
else if (nch == 3)
res = (validate_listmaker(CHILD(tree, 1))
res = (validate_testlist_comp(CHILD(tree, 1))
&& validate_ntype(CHILD(tree, 2), RSQB));
else {
res = 0;
@ -2412,7 +2346,7 @@ validate_atom(node *tree)
&& validate_ntype(CHILD(tree, nch - 1), RBRACE));
if (res && (nch == 3))
res = validate_dictsetmaker(CHILD(tree, 1));
res = validate_dictorsetmaker(CHILD(tree, 1));
break;
case NAME:
case NUMBER:
@ -2436,25 +2370,26 @@ validate_atom(node *tree)
}
/* listmaker:
* test ( list_for | (',' test)* [','] )
/* testlist_comp:
* test ( comp_for | (',' test)* [','] )
*/
static int
validate_listmaker(node *tree)
validate_testlist_comp(node *tree)
{
int nch = NCH(tree);
int ok = nch;
if (nch == 0)
err_string("missing child nodes of listmaker");
else
err_string("missing child nodes of testlist_comp");
else {
ok = validate_test(CHILD(tree, 0));
}
/*
* list_for | (',' test)* [',']
* comp_for | (',' test)* [',']
*/
if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for)
ok = validate_list_for(CHILD(tree, 1));
if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for)
ok = validate_comp_for(CHILD(tree, 1));
else {
/* (',' test)* [','] */
int i = 1;
@ -2467,45 +2402,7 @@ validate_listmaker(node *tree)
ok = validate_comma(CHILD(tree, i));
else if (i != nch) {
ok = 0;
err_string("illegal trailing nodes for listmaker");
}
}
return ok;
}
/* testlist_gexp:
* test ( gen_for | (',' test)* [','] )
*/
static int
validate_testlist_gexp(node *tree)
{
int nch = NCH(tree);
int ok = nch;
if (nch == 0)
err_string("missing child nodes of testlist_gexp");
else {
ok = validate_test(CHILD(tree, 0));
}
/*
* gen_for | (',' test)* [',']
*/
if (nch == 2 && TYPE(CHILD(tree, 1)) == gen_for)
ok = validate_gen_for(CHILD(tree, 1));
else {
/* (',' test)* [','] */
int i = 1;
while (ok && nch - i >= 2) {
ok = (validate_comma(CHILD(tree, i))
&& validate_test(CHILD(tree, i+1)));
i += 2;
}
if (ok && i == nch-1)
ok = validate_comma(CHILD(tree, i));
else if (i != nch) {
ok = 0;
err_string("illegal trailing nodes for testlist_gexp");
err_string("illegal trailing nodes for testlist_comp");
}
}
return ok;
@ -2596,10 +2493,10 @@ validate_lambdef(node *tree)
static int
validate_old_lambdef(node *tree)
validate_lambdef_nocond(node *tree)
{
int nch = NCH(tree);
int res = (validate_ntype(tree, old_lambdef)
int res = (validate_ntype(tree, lambdef_nocond)
&& ((nch == 3) || (nch == 4))
&& validate_name(CHILD(tree, 0), "lambda")
&& validate_colon(CHILD(tree, nch - 2))
@ -2608,7 +2505,7 @@ validate_old_lambdef(node *tree)
if (res && (nch == 4))
res = validate_varargslist(CHILD(tree, 1));
else if (!res && !PyErr_Occurred())
(void) validate_numnodes(tree, 3, "old_lambdef");
(void) validate_numnodes(tree, 3, "lambdef_nocond");
return (res);
}
@ -2633,7 +2530,7 @@ validate_arglist(node *tree)
for (i=0; i<nch; i++) {
if (TYPE(CHILD(tree, i)) == argument) {
node *ch = CHILD(tree, i);
if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == gen_for) {
if (NCH(ch) == 2 && TYPE(CHILD(ch, 1)) == comp_for) {
err_string("need '(', ')' for generator expression");
return 0;
}
@ -2700,7 +2597,7 @@ validate_arglist(node *tree)
/* argument:
*
* [test '='] test [gen_for]
* [test '='] test [comp_for]
*/
static int
validate_argument(node *tree)
@ -2711,7 +2608,7 @@ validate_argument(node *tree)
&& validate_test(CHILD(tree, 0)));
if (res && (nch == 2))
res = validate_gen_for(CHILD(tree, 1));
res = validate_comp_for(CHILD(tree, 1));
else if (res && (nch == 3))
res = (validate_equal(CHILD(tree, 1))
&& validate_test(CHILD(tree, 2)));
@ -2853,10 +2750,10 @@ validate_exprlist(node *tree)
static int
validate_dictsetmaker(node *tree)
validate_dictorsetmaker(node *tree)
{
int nch = NCH(tree);
int res = (validate_ntype(tree, dictsetmaker)
int res = (validate_ntype(tree, dictorsetmaker)
&& (nch >= 3)
&& validate_test(CHILD(tree, 0))
&& validate_colon(CHILD(tree, 1))

2
Modules/symtablemodule.c
View File

@ -33,7 +33,7 @@ symtable_symtable(PyObject *self, PyObject *args)
st = Py_SymtableString(str, filename, start);
if (st == NULL)
return NULL;
t = st->st_symbols;
t = st->st_blocks;
Py_INCREF(t);
PyMem_Free((void *)st->st_future);
PySymtable_Free(st);

1
Parser/Python.asdl
View File

@ -56,6 +56,7 @@ module Python version "$Revision$"
| Dict(expr* keys, expr* values)
| Set(expr* elts)
| ListComp(expr elt, comprehension* generators)
| SetComp(expr elt, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur
| Yield(expr? value)

45
Python/Python-ast.c
View File

@ -192,6 +192,11 @@ static char *ListComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *SetComp_type;
static char *SetComp_fields[]={
"elt",
"generators",
};
static PyTypeObject *GeneratorExp_type;
static char *GeneratorExp_fields[]={
"elt",
@ -543,6 +548,8 @@ static int init_types(void)
if (!Set_type) return 0;
ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
if (!ListComp_type) return 0;
SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
if (!SetComp_type) return 0;
GeneratorExp_type = make_type("GeneratorExp", expr_type,
GeneratorExp_fields, 2);
if (!GeneratorExp_type) return 0;
@ -1418,6 +1425,27 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
return p;
}
expr_ty
SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
*arena)
{
expr_ty p;
if (!elt) {
PyErr_SetString(PyExc_ValueError,
"field elt is required for SetComp");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = SetComp_kind;
p->v.SetComp.elt = elt;
p->v.SetComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
}
expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
PyArena *arena)
@ -2416,6 +2444,21 @@ ast2obj_expr(void* _o)
goto failed;
Py_DECREF(value);
break;
case SetComp_kind:
result = PyType_GenericNew(SetComp_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_expr(o->v.SetComp.elt);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.SetComp.generators,
ast2obj_comprehension);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
Py_DECREF(value);
break;
case GeneratorExp_kind:
result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
if (!result) goto failed;
@ -3120,6 +3163,8 @@ init_ast(void)
if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return;
if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0)
return;
if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0)
return;
if (PyDict_SetItemString(d, "GeneratorExp",
(PyObject*)GeneratorExp_type) < 0) return;
if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return;

366
Python/ast.c
View File

@ -27,7 +27,6 @@ static stmt_ty ast_for_stmt(struct compiling *, const node *);
static asdl_seq *ast_for_suite(struct compiling *, const node *);
static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
@ -41,6 +40,10 @@ static PyObject *parsestrplus(struct compiling *, const node *n,
#define LINENO(n) ((n)->n_lineno)
#endif
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
static identifier
new_identifier(const char* n, PyArena *arena) {
PyObject* id = PyString_InternFromString(n);
@ -231,7 +234,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
case eval_input: {
expr_ty testlist_ast;
/* XXX Why not gen_for here? */
/* XXX Why not comp_for here? */
testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
if (!testlist_ast)
goto error;
@ -530,19 +533,14 @@ seq_for_testlist(struct compiling *c, const node *n)
asdl_seq *seq;
expr_ty expression;
int i;
assert(TYPE(n) == testlist
|| TYPE(n) == listmaker
|| TYPE(n) == testlist_gexp
|| TYPE(n) == testlist_safe
|| TYPE(n) == testlist1
);
assert(TYPE(n) == testlist || TYPE(n) == testlist_comp);
seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!seq)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test);
assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond);
expression = ast_for_expr(c, CHILD(n, i));
if (!expression)
@ -1022,7 +1020,8 @@ ast_for_funcdef(struct compiling *c, const node *n)
static expr_ty
ast_for_lambdef(struct compiling *c, const node *n)
{
/* lambdef: 'lambda' [varargslist] ':' test */
/* lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
arguments_ty args;
expr_ty expression;
@ -1067,190 +1066,34 @@ ast_for_ifexpr(struct compiling *c, const node *n)
c->c_arena);
}
/* XXX(nnorwitz): the listcomp and genexpr code should be refactored
so there is only a single version. Possibly for loops can also re-use
the code.
*/
/* Count the number of 'for' loop in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_fors(const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_list_for:
n_fors++;
REQ(ch, list_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
else
return n_fors;
count_list_iter:
REQ(ch, list_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == list_for)
goto count_list_for;
else if (TYPE(ch) == list_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_list_iter;
}
else
return n_fors;
}
/* Should never be reached */
PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");
return -1;
}
/* Count the number of 'if' statements in a list comprehension.
Helper for ast_for_listcomp().
*/
static int
count_list_ifs(const node *n)
{
int n_ifs = 0;
count_list_iter:
REQ(n, list_iter);
if (TYPE(CHILD(n, 0)) == list_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, list_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
n = CHILD(n, 2);
goto count_list_iter;
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
/* listmaker: test ( list_for | (',' test)* [','] )
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_iter: list_for | list_if
list_if: 'if' test [list_iter]
testlist_safe: test [(',' test)+ [',']]
*/
expr_ty elt;
asdl_seq *listcomps;
int i, n_fors;
node *ch;
REQ(n, listmaker);
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_list_fors(n);
if (n_fors == -1)
return NULL;
listcomps = asdl_seq_new(n_fors, c->c_arena);
if (!listcomps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty lc;
asdl_seq *t;
expr_ty expression;
node *for_ch;
REQ(ch, list_for);
for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
return NULL;
expression = ast_for_testlist(c, CHILD(ch, 3));
if (!expression)
return NULL;
/* Check the # of children rather than the length of t, since
[x for x, in ... ] has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1)
lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL,
c->c_arena);
else
lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!lc)
return NULL;
if (NCH(ch) == 5) {
int j, n_ifs;
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_list_ifs(ch);
if (n_ifs == -1)
return NULL;
ifs = asdl_seq_new(n_ifs, c->c_arena);
if (!ifs)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, list_iter);
ch = CHILD(ch, 0);
REQ(ch, list_if);
asdl_seq_SET(ifs, j, ast_for_expr(c, CHILD(ch, 1)));
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
}
/* on exit, must guarantee that ch is a list_for */
if (TYPE(ch) == list_iter)
ch = CHILD(ch, 0);
lc->ifs = ifs;
}
asdl_seq_SET(listcomps, i, lc);
}
return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/*
Count the number of 'for' loops in a generator expression.
Count the number of 'for' loops in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_fors(const node *n)
count_comp_fors(const node *n)
{
int n_fors = 0;
node *ch = CHILD(n, 1);
count_gen_for:
count_comp_for:
n_fors++;
REQ(ch, gen_for);
REQ(ch, comp_for);
if (NCH(ch) == 5)
ch = CHILD(ch, 4);
else
return n_fors;
count_gen_iter:
REQ(ch, gen_iter);
count_comp_iter:
REQ(ch, comp_iter);
ch = CHILD(ch, 0);
if (TYPE(ch) == gen_for)
goto count_gen_for;
else if (TYPE(ch) == gen_if) {
if (TYPE(ch) == comp_for)
goto count_comp_for;
else if (TYPE(ch) == comp_if) {
if (NCH(ch) == 3) {
ch = CHILD(ch, 2);
goto count_gen_iter;
goto count_comp_iter;
}
else
return n_fors;
@ -1258,26 +1101,26 @@ count_gen_fors(const node *n)
/* Should never be reached */
PyErr_SetString(PyExc_SystemError,
"logic error in count_gen_fors");
"logic error in count_comp_fors");
return -1;
}
/* Count the number of 'if' statements in a generator expression.
/* Count the number of 'if' statements in a comprehension.
Helper for ast_for_genexp().
Helper for ast_for_comprehension().
*/
static int
count_gen_ifs(const node *n)
count_comp_ifs(const node *n)
{
int n_ifs = 0;
while (1) {
REQ(n, gen_iter);
if (TYPE(CHILD(n, 0)) == gen_for)
REQ(n, comp_iter);
if (TYPE(CHILD(n, 0)) == comp_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, gen_if);
REQ(n, comp_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
@ -1285,40 +1128,38 @@ count_gen_ifs(const node *n)
}
}
/* TODO(jhylton): Combine with list comprehension code? */
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
ast_for_comprehension(struct compiling *c, const node *n, int type)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] )
argument: [test '='] test [gen_for] # Really [keyword '='] test */
/* testlist_comp: test ( comp_for | (',' test)* [','] )
argument: [test '='] test [comp_for] # Really [keyword '='] test */
expr_ty elt;
asdl_seq *genexps;
asdl_seq *comps;
int i, n_fors;
node *ch;
assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument));
assert(NCH(n) > 1);
elt = ast_for_expr(c, CHILD(n, 0));
if (!elt)
return NULL;
n_fors = count_gen_fors(n);
n_fors = count_comp_fors(n);
if (n_fors == -1)
return NULL;
genexps = asdl_seq_new(n_fors, c->c_arena);
if (!genexps)
comps = asdl_seq_new(n_fors, c->c_arena);
if (!comps)
return NULL;
ch = CHILD(n, 1);
for (i = 0; i < n_fors; i++) {
comprehension_ty ge;
comprehension_ty comp;
asdl_seq *t;
expr_ty expression;
node *for_ch;
REQ(ch, gen_for);
REQ(ch, comp_for);
for_ch = CHILD(ch, 1);
t = ast_for_exprlist(c, for_ch, Store);
@ -1331,14 +1172,14 @@ ast_for_genexp(struct compiling *c, const node *n)
/* Check the # of children rather than the length of t, since
(x for x, in ...) has 1 element in t, but still requires a Tuple. */
if (NCH(for_ch) == 1)
ge = comprehension((expr_ty)asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
comp = comprehension((expr_ty)asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
else
ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
comp = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset,
c->c_arena),
expression, NULL, c->c_arena);
if (!ge)
if (!comp)
return NULL;
if (NCH(ch) == 5) {
@ -1346,7 +1187,7 @@ ast_for_genexp(struct compiling *c, const node *n)
asdl_seq *ifs;
ch = CHILD(ch, 4);
n_ifs = count_gen_ifs(ch);
n_ifs = count_comp_ifs(ch);
if (n_ifs == -1)
return NULL;
@ -1355,9 +1196,9 @@ ast_for_genexp(struct compiling *c, const node *n)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(ch, gen_iter);
REQ(ch, comp_iter);
ch = CHILD(ch, 0);
REQ(ch, gen_if);
REQ(ch, comp_if);
expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression)
@ -1366,22 +1207,52 @@ ast_for_genexp(struct compiling *c, const node *n)
if (NCH(ch) == 3)
ch = CHILD(ch, 2);
}
/* on exit, must guarantee that ch is a gen_for */
if (TYPE(ch) == gen_iter)
/* on exit, must guarantee that ch is a comp_for */
if (TYPE(ch) == comp_iter)
ch = CHILD(ch, 0);
ge->ifs = ifs;
comp->ifs = ifs;
}
asdl_seq_SET(genexps, i, ge);
asdl_seq_SET(comps, i, comp);
}
return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_LISTCOMP)
return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
}
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument));
return ast_for_comprehension(c, n, COMP_GENEXP);
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp));
return ast_for_comprehension(c, n, COMP_LISTCOMP);
}
static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_comprehension(c, n, COMP_SETCOMP);
}
static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
/* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']'
| '{' [dictsetmaker] '}' | NAME | NUMBER | STRING+
/* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
| '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
*/
node *ch = CHILD(n, 0);
int bytesmode = 0;
@ -1420,18 +1291,19 @@ ast_for_atom(struct compiling *c, const node *n)
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for))
/* testlist_comp: test ( comp_for | (',' test)* [','] ) */
if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for))
return ast_for_genexp(c, ch);
return ast_for_testlist_gexp(c, ch);
return ast_for_testlist(c, ch);
case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, listmaker);
REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts)
@ -1442,27 +1314,32 @@ ast_for_atom(struct compiling *c, const node *n)
else
return ast_for_listcomp(c, ch);
case LBRACE: {
/* dictsetmaker: test ':' test (',' test ':' test)* [','] |
* test (',' test)* [','] */
/* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
* test (gen_for | (',' test)* [',']) */
int i, size;
asdl_seq *keys, *values;
ch = CHILD(n, 1);
if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) {
/* it's a set */
if (TYPE(ch) == RBRACE) {
/* it's an empty dict */
return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
} else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
/* it's a simple set */
size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
keys = asdl_seq_new(size, c->c_arena);
if (!keys)
asdl_seq *elts = asdl_seq_new(size, c->c_arena);
if (!elts)
return NULL;
for (i = 0; i < NCH(ch); i += 2) {
expr_ty expression;
expression = ast_for_expr(c, CHILD(ch, i));
if (!expression)
return NULL;
asdl_seq_SET(keys, i / 2, expression);
asdl_seq_SET(elts, i / 2, expression);
}
return Set(keys, LINENO(n), n->n_col_offset, c->c_arena);
return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
} else if (TYPE(CHILD(ch, 1)) == comp_for) {
/* it's a set comprehension */
return ast_for_setcomp(c, ch);
} else {
/* it's a dict */
size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
@ -1790,6 +1667,7 @@ ast_for_expr(struct compiling *c, const node *n)
{
/* handle the full range of simple expressions
test: or_test ['if' or_test 'else' test] | lambdef
test_nocond: or_test | lambdef_nocond
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
@ -1802,15 +1680,6 @@ ast_for_expr(struct compiling *c, const node *n)
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ('**' factor)*
As well as modified versions that exist for backward compatibility,
to explicitly allow:
[ x for x in lambda: 0, lambda: 1 ]
(which would be ambiguous without these extra rules)
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [vararglist] ':' old_test
*/
asdl_seq *seq;
@ -1819,9 +1688,9 @@ ast_for_expr(struct compiling *c, const node *n)
loop:
switch (TYPE(n)) {
case test:
case old_test:
case test_nocond:
if (TYPE(CHILD(n, 0)) == lambdef ||
TYPE(CHILD(n, 0)) == old_lambdef)
TYPE(CHILD(n, 0)) == lambdef_nocond)
return ast_for_lambdef(c, CHILD(n, 0));
else if (NCH(n) > 1)
return ast_for_ifexpr(c, n);
@ -1947,7 +1816,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
/*
arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
| '**' test)
argument: [test '='] test [gen_for] # Really [keyword '='] test
argument: [test '='] test [comp_for] # Really [keyword '='] test
*/
int i, nargs, nkeywords, ngens;
@ -1965,7 +1834,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
if (TYPE(ch) == argument) {
if (NCH(ch) == 1)
nargs++;
else if (TYPE(CHILD(ch, 1)) == gen_for)
else if (TYPE(CHILD(ch, 1)) == comp_for)
ngens++;
else
nkeywords++;
@ -2005,7 +1874,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
return NULL;
asdl_seq_SET(args, nargs++, e);
}
else if (TYPE(CHILD(ch, 1)) == gen_for) {
else if (TYPE(CHILD(ch, 1)) == comp_for) {
e = ast_for_genexp(c, ch);
if (!e)
return NULL;
@ -2057,18 +1926,16 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
static expr_ty
ast_for_testlist(struct compiling *c, const node* n)
{
/* testlist_gexp: test (',' test)* [','] */
/* testlist_comp: test (comp_for | (',' test)* [',']) */
/* testlist: test (',' test)* [','] */
/* testlist_safe: test (',' test)+ [','] */
/* testlist1: test (',' test)* */
assert(NCH(n) > 0);
if (TYPE(n) == testlist_gexp) {
if (TYPE(n) == testlist_comp) {
if (NCH(n) > 1)
assert(TYPE(CHILD(n, 1)) != gen_for);
assert(TYPE(CHILD(n, 1)) != comp_for);
}
else {
assert(TYPE(n) == testlist ||
TYPE(n) == testlist_safe ||
TYPE(n) == testlist1);
}
if (NCH(n) == 1)
@ -2081,17 +1948,6 @@ ast_for_testlist(struct compiling *c, const node* n)
}
}
static expr_ty
ast_for_testlist_gexp(struct compiling *c, const node* n)
{
/* testlist_gexp: test ( gen_for | (',' test)* [','] ) */
/* argument: test [ gen_for ] */
assert(TYPE(n) == testlist_gexp || TYPE(n) == argument);
if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for)
return ast_for_genexp(c, n);
return ast_for_testlist(c, n);
}
static stmt_ty
ast_for_expr_stmt(struct compiling *c, const node *n)
{

12
Python/ceval.c
View File

@ -1241,6 +1241,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
}
break;
case SET_ADD:
w = POP();
v = POP();
err = PySet_Add(v, w);
Py_DECREF(v);
Py_DECREF(w);
if (err == 0) {
PREDICT(JUMP_ABSOLUTE);
continue;
}
break;
case INPLACE_POWER:
w = POP();
v = TOP();

295
Python/compile.c
View File

@ -39,6 +39,10 @@ int Py_OptimizeFlag = 0;
#define DEFAULT_CODE_SIZE 128
#define DEFAULT_LNOTAB_SIZE 16
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
struct instr {
unsigned i_jabs : 1;
unsigned i_jrel : 1;
@ -360,7 +364,7 @@ dictbytype(PyObject *src, int scope_type, int flag, int offset)
while (PyDict_Next(src, &pos, &k, &v)) {
/* XXX this should probably be a macro in symtable.h */
assert(PyInt_Check(v));
scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK;
scope = (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
if (scope == scope_type || PyInt_AS_LONG(v) & flag) {
PyObject *tuple, *item = PyInt_FromLong(i);
@ -673,6 +677,7 @@ opcode_stack_effect(int opcode, int oparg)
case UNARY_INVERT:
return 0;
case SET_ADD:
case LIST_APPEND:
return -2;
@ -2724,15 +2729,31 @@ compiler_call_helper(struct compiler *c,
return 1;
}
/* List and set comprehensions and generator expressions work by creating a
nested function to perform the actual iteration. This means that the
iteration variables don't leak into the current scope.
The defined function is called immediately following its definition, with the
result of that call being the result of the expression.
The LC/SC version returns the populated container, while the GE version is
flagged in symtable.c as a generator, so it returns the generator object
when the function is called.
This code *knows* that the loop cannot contain break, continue, or return,
so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
Possible cleanups:
- iterate over the generator sequence instead of using recursion
*/
static int
compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
asdl_seq *generators, int gen_index,
expr_ty elt)
compiler_comprehension_generator(struct compiler *c, PyObject *tmpname,
asdl_seq *generators, int gen_index,
expr_ty elt, int type)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty l;
comprehension_ty gen;
basicblock *start, *anchor, *skip, *if_cleanup;
int i, n;
@ -2742,104 +2763,11 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
anchor = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL)
return 0;
l = (comprehension_ty)asdl_seq_GET(generators, gen_index);
VISIT(c, expr, l->iter);
ADDOP(c, GET_ITER);
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, l->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(l->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i);
VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
ADDOP(c, POP_TOP);
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_listcomp_generator(c, tmpname,
generators, gen_index, elt))
anchor == NULL)
return 0;
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
compiler_use_next_block(c, skip);
}
for (i = 0; i < n; i++) {
ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0)
compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP);
}
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
/* delete the temporary list name added to locals */
if (gen_index == 1)
if (!compiler_nameop(c, tmpname, Del))
return 0;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
return 1;
}
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
identifier tmp;
int rc = 0;
asdl_seq *generators = e->v.ListComp.generators;
assert(e->kind == ListComp_kind);
tmp = compiler_new_tmpname(c);
if (!tmp)
return 0;
ADDOP_I(c, BUILD_LIST, 0);
ADDOP(c, DUP_TOP);
if (compiler_nameop(c, tmp, Store))
rc = compiler_listcomp_generator(c, tmp, generators, 0,
e->v.ListComp.elt);
Py_DECREF(tmp);
return rc;
}
static int
compiler_genexp_generator(struct compiler *c,
asdl_seq *generators, int gen_index,
expr_ty elt)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
comprehension_ty ge;
basicblock *start, *anchor, *skip, *if_cleanup, *end;
int i, n;
start = compiler_new_block(c);
skip = compiler_new_block(c);
if_cleanup = compiler_new_block(c);
anchor = compiler_new_block(c);
end = compiler_new_block(c);
if (start == NULL || skip == NULL || if_cleanup == NULL ||
anchor == NULL || end == NULL)
return 0;
ge = (comprehension_ty)asdl_seq_GET(generators, gen_index);
ADDOP_JREL(c, SETUP_LOOP, end);
if (!compiler_push_fblock(c, LOOP, start))
return 0;
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1;
@ -2847,18 +2775,18 @@ compiler_genexp_generator(struct compiler *c,
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, ge->iter);
VISIT(c, expr, gen->iter);
ADDOP(c, GET_ITER);
}
compiler_use_next_block(c, start);
ADDOP_JREL(c, FOR_ITER, anchor);
NEXT_BLOCK(c);
VISIT(c, expr, ge->target);
VISIT(c, expr, gen->target);
/* XXX this needs to be cleaned up...a lot! */
n = asdl_seq_LEN(ge->ifs);
n = asdl_seq_LEN(gen->ifs);
for (i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i);
expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
VISIT(c, expr, e);
ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
NEXT_BLOCK(c);
@ -2866,14 +2794,35 @@ compiler_genexp_generator(struct compiler *c,
}
if (++gen_index < asdl_seq_LEN(generators))
if (!compiler_genexp_generator(c, generators, gen_index, elt))
return 0;
if (!compiler_comprehension_generator(c, tmpname,
generators, gen_index,
elt, type))
return 0;
/* only append after the last 'for' generator */
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
/* comprehension specific code */
switch (type) {
case COMP_GENEXP:
VISIT(c, expr, elt);
ADDOP(c, YIELD_VALUE);
ADDOP(c, POP_TOP);
break;
case COMP_LISTCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, LIST_APPEND);
break;
case COMP_SETCOMP:
if (!compiler_nameop(c, tmpname, Load))
return 0;
VISIT(c, expr, elt);
ADDOP(c, SET_ADD);
break;
default:
return 0;
}
compiler_use_next_block(c, skip);
}
@ -2881,52 +2830,116 @@ compiler_genexp_generator(struct compiler *c,
ADDOP_I(c, JUMP_FORWARD, 1);
if (i == 0)
compiler_use_next_block(c, if_cleanup);
ADDOP(c, POP_TOP);
}
ADDOP_JABS(c, JUMP_ABSOLUTE, start);
compiler_use_next_block(c, anchor);
ADDOP(c, POP_BLOCK);
compiler_pop_fblock(c, LOOP, start);
compiler_use_next_block(c, end);
return 1;
}
static int
compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
asdl_seq *generators, expr_ty elt)
{
PyCodeObject *co = NULL;
identifier tmp = NULL;
expr_ty outermost_iter;
outermost_iter = ((comprehension_ty)
asdl_seq_GET(generators, 0))->iter;
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
goto error;
if (type != COMP_GENEXP) {
tmp = compiler_new_tmpname(c);
if (!tmp)
goto error_in_scope;
ADDOP_I(c, (type == COMP_LISTCOMP ?
BUILD_LIST : BUILD_SET), 0);
ADDOP(c, DUP_TOP);
if (!compiler_nameop(c, tmp, Store))
goto error_in_scope;
}
if (!compiler_comprehension_generator(c, tmp, generators, 0, elt, type))
goto error_in_scope;
if (type != COMP_GENEXP) {
ADDOP(c, RETURN_VALUE);
}
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
goto error;
if (!compiler_make_closure(c, co, 0))
goto error;
Py_DECREF(co);
Py_XDECREF(tmp);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
error_in_scope:
compiler_exit_scope(c);
error:
Py_XDECREF(co);
Py_XDECREF(tmp);
return 0;
}
static int
compiler_genexp(struct compiler *c, expr_ty e)
{
static identifier name;
PyCodeObject *co;
expr_ty outermost_iter = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators,
0)))->iter;
if (!name) {
name = PyString_FromString("<genexpr>");
name = PyString_FromString("<genexp>");
if (!name)
return 0;
}
if (!compiler_enter_scope(c, name, (void *)e, e->lineno))
return 0;
compiler_genexp_generator(c, e->v.GeneratorExp.generators, 0,
e->v.GeneratorExp.elt);
co = assemble(c, 1);
compiler_exit_scope(c);
if (co == NULL)
return 0;
compiler_make_closure(c, co, 0);
Py_DECREF(co);
VISIT(c, expr, outermost_iter);
ADDOP(c, GET_ITER);
ADDOP_I(c, CALL_FUNCTION, 1);
return 1;
assert(e->kind == GeneratorExp_kind);
return compiler_comprehension(c, e, COMP_GENEXP, name,
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt);
}
static int
compiler_listcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<listcomp>");
if (!name)
return 0;
}
assert(e->kind == ListComp_kind);
return compiler_comprehension(c, e, COMP_LISTCOMP, name,
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
compiler_setcomp(struct compiler *c, expr_ty e)
{
static identifier name;
if (!name) {
name = PyString_FromString("<setcomp>");
if (!name)
return 0;
}
assert(e->kind == SetComp_kind);
return compiler_comprehension(c, e, COMP_SETCOMP, name,
e->v.SetComp.generators,
e->v.SetComp.elt);
}
static int
compiler_visit_keyword(struct compiler *c, keyword_ty k)
{
@ -3145,10 +3158,12 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
VISIT_SEQ(c, expr, e->v.Set.elts);
ADDOP_I(c, BUILD_SET, n);
break;
case ListComp_kind:
return compiler_listcomp(c, e);
case GeneratorExp_kind:
return compiler_genexp(c, e);
case ListComp_kind:
return compiler_listcomp(c, e);
case SetComp_kind:
return compiler_setcomp(c, e);
case Yield_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'yield' outside function");

986
Python/graminit.c
View File

File diff suppressed because it is too large Load Diff

347
Python/symtable.c
View File

@ -76,7 +76,7 @@ PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_generator = 0;
ste->ste_returns_value = 0;
if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0)
if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
goto fail;
return ste;
@ -172,6 +172,8 @@ static int symtable_exit_block(struct symtable *st, void *ast);
static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
static int symtable_visit_expr(struct symtable *st, expr_ty s);
static int symtable_visit_genexp(struct symtable *st, expr_ty s);
static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
static int symtable_visit_arguments(struct symtable *st, arguments_ty);
static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
static int symtable_visit_alias(struct symtable *st, alias_ty);
@ -186,7 +188,8 @@ static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
static identifier top = NULL, lambda = NULL, genexpr = NULL;
static identifier top = NULL, lambda = NULL, genexpr = NULL,
listcomp = NULL, setcomp = NULL;
#define GET_IDENTIFIER(VAR) \
((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR)))
@ -204,14 +207,13 @@ symtable_new(void)
return NULL;
st->st_filename = NULL;
st->st_symbols = NULL;
st->st_blocks = NULL;
if ((st->st_stack = PyList_New(0)) == NULL)
goto fail;
if ((st->st_symbols = PyDict_New()) == NULL)
if ((st->st_blocks = PyDict_New()) == NULL)
goto fail;
st->st_cur = NULL;
st->st_tmpname = 0;
st->st_private = NULL;
return st;
fail:
@ -230,6 +232,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
return st;
st->st_filename = filename;
st->st_future = future;
/* Make the initial symbol information gathering pass */
if (!GET_IDENTIFIER(top) ||
!symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) {
PySymtable_Free(st);
@ -238,7 +241,6 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
st->st_top = st->st_cur;
st->st_cur->ste_unoptimized = OPT_TOPLEVEL;
/* Any other top-level initialization? */
switch (mod->kind) {
case Module_kind:
seq = mod->v.Module.body;
@ -267,6 +269,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
PySymtable_Free(st);
return NULL;
}
/* Make the second symbol analysis pass */
if (symtable_analyze(st))
return st;
PySymtable_Free(st);
@ -280,7 +283,7 @@ PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
void
PySymtable_Free(struct symtable *st)
{
Py_XDECREF(st->st_symbols);
Py_XDECREF(st->st_blocks);
Py_XDECREF(st->st_stack);
PyMem_Free((void *)st);
}
@ -293,7 +296,7 @@ PySymtable_Lookup(struct symtable *st, void *key)
k = PyLong_FromVoidPtr(key);
if (k == NULL)
return NULL;
v = PyDict_GetItem(st->st_symbols, k);
v = PyDict_GetItem(st->st_blocks, k);
if (v) {
assert(PySTEntry_Check(v));
Py_INCREF(v);
@ -314,7 +317,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
if (!v)
return 0;
assert(PyInt_Check(v));
return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK;
return (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
}
@ -325,7 +328,7 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
it determines which local variables are cell variables; they provide
bindings that are used for free variables in enclosed blocks.
There are also two kinds of free variables, implicit and explicit. An
There are also two kinds of global variables, implicit and explicit. An
explicit global is declared with the global statement. An implicit
global is a free variable for which the compiler has found no binding
in an enclosing function scope. The implicit global is either a global
@ -337,24 +340,30 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
TODO(jhylton): Discuss nonlocal
The symbol table requires two passes to determine the scope of each name.
The first pass collects raw facts from the AST: the name is a parameter
here, the name is used by not defined here, etc. The second pass analyzes
these facts during a pass over the PySTEntryObjects created during pass 1.
The first pass collects raw facts from the AST via the symtable_visit_*
functions: the name is a parameter here, the name is used but not defined
here, etc. The second pass analyzes these facts during a pass over the
PySTEntryObjects created during pass 1.
When a function is entered during the second pass, the parent passes
the set of all name bindings visible to its children. These bindings
are used to determine if the variable is free or an implicit global.
are used to determine if non-local variables are free or implicit globals.
After doing the local analysis, it analyzes each of its child blocks
using an updated set of name bindings.
using an updated set of name bindings.
The children update the free variable set. If a local variable is free
in a child, the variable is marked as a cell. The current function must
provide runtime storage for the variable that may outlive the function's
frame. Cell variables are removed from the free set before the analyze
function returns to its parent.
The children update the free variable set. If a local variable is added to
the free variable set by the child, the variable is marked as a cell. The
function object being defined must provide runtime storage for the variable
that may outlive the function's frame. Cell variables are removed from the
free set before the analyze function returns to its parent.
The sets of bound and free variables are implemented as dictionaries
mapping strings to None.
During analysis, the names are:
symbols: dict mapping from symbol names to flag values (including offset scope values)
scopes: dict mapping from symbol names to scope values (no offset)
local: set of all symbol names local to the current scope
bound: set of all symbol names local to a containing function scope
free: set of all symbol names referenced but not bound in child scopes
global: set of all symbol names explicitly declared as global
*/
#define SET_SCOPE(DICT, NAME, I) { \
@ -375,14 +384,14 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
*/
static int
analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
PyObject *bound, PyObject *local, PyObject *free,
PyObject *global)
{
if (flags & DEF_GLOBAL) {
if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and global",
"name '%s' is parameter and global",
PyString_AS_STRING(name));
return 0;
}
@ -392,41 +401,37 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
PyString_AS_STRING(name));
return 0;
}
SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
if (PyDict_SetItem(global, name, Py_None) < 0)
SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
if (PySet_Add(global, name) < 0)
return 0;
if (bound && (PySet_Discard(bound, name) < 0))
return 0;
if (bound && PyDict_GetItem(bound, name)) {
if (PyDict_DelItem(bound, name) < 0)
return 0;
}
return 1;
}
if (flags & DEF_NONLOCAL) {
if (flags & DEF_PARAM) {
PyErr_Format(PyExc_SyntaxError,
"name '%s' is local and nonlocal",
"name '%s' is parameter and nonlocal",
PyString_AS_STRING(name));
return 0;
}
if (!PyDict_GetItem(bound, name)) {
if (!PySet_Contains(bound, name)) {
PyErr_Format(PyExc_SyntaxError,
"no binding for nonlocal '%s' found",
PyString_AS_STRING(name));
return 0;
}
SET_SCOPE(dict, name, FREE);
SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0;
return PySet_Add(free, name) >= 0;
}
if (flags & DEF_BOUND) {
SET_SCOPE(dict, name, LOCAL);
if (PyDict_SetItem(local, name, Py_None) < 0)
SET_SCOPE(scopes, name, LOCAL);
if (PySet_Add(local, name) < 0)
return 0;
if (PySet_Discard(global, name) < 0)
return 0;
if (PyDict_GetItem(global, name)) {
if (PyDict_DelItem(global, name) < 0)
return 0;
}
return 1;
}
/* If an enclosing block has a binding for this name, it
@ -434,21 +439,21 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
Note that having a non-NULL bound implies that the block
is nested.
*/
if (bound && PyDict_GetItem(bound, name)) {
SET_SCOPE(dict, name, FREE);
if (bound && PySet_Contains(bound, name)) {
SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1;
return PyDict_SetItem(free, name, Py_None) >= 0;
return PySet_Add(free, name) >= 0;
}
/* If a parent has a global statement, then call it global
explicit? It could also be global implicit.
*/
if (global && PyDict_GetItem(global, name)) {
SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
if (global && PySet_Contains(global, name)) {
SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
return 1;
}
if (ste->ste_nested)
ste->ste_free = 1;
SET_SCOPE(dict, name, GLOBAL_IMPLICIT);
SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
return 1;
}
@ -463,35 +468,35 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
*/
static int
analyze_cells(PyObject *scope, PyObject *free)
analyze_cells(PyObject *scopes, PyObject *free)
{
PyObject *name, *v, *w;
PyObject *name, *v, *v_cell;
int success = 0;
Py_ssize_t pos = 0;
w = PyInt_FromLong(CELL);
if (!w)
v_cell = PyInt_FromLong(CELL);
if (!v_cell)
return 0;
while (PyDict_Next(scope, &pos, &name, &v)) {
long flags;
while (PyDict_Next(scopes, &pos, &name, &v)) {
long scope;
assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v);
if (flags != LOCAL)
scope = PyInt_AS_LONG(v);
if (scope != LOCAL)
continue;
if (!PyDict_GetItem(free, name))
if (!PySet_Contains(free, name))
continue;
/* Replace LOCAL with CELL for this name, and remove
from free. It is safe to replace the value of name
in the dict, because it will not cause a resize.
*/
if (PyDict_SetItem(scope, name, w) < 0)
if (PyDict_SetItem(scopes, name, v_cell) < 0)
goto error;
if (!PyDict_DelItem(free, name) < 0)
if (PySet_Discard(free, name) < 0)
goto error;
}
success = 1;
error:
Py_DECREF(w);
Py_DECREF(v_cell);
return success;
}
@ -526,77 +531,91 @@ check_unoptimized(const PySTEntryObject* ste) {
return 0;
}
/* Enter the final scope information into the st_symbols dict.
/* Enter the final scope information into the ste_symbols dict.
*
* All arguments are dicts. Modifies symbols, others are read-only.
*/
static int
update_symbols(PyObject *symbols, PyObject *scope,
update_symbols(PyObject *symbols, PyObject *scopes,
PyObject *bound, PyObject *free, int classflag)
{
PyObject *name, *v, *u, *w, *free_value = NULL;
PyObject *name = NULL, *itr = NULL;
PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
Py_ssize_t pos = 0;
/* Update scope information for all symbols in this scope */
while (PyDict_Next(symbols, &pos, &name, &v)) {
long i, flags;
long scope, flags;
assert(PyInt_Check(v));
flags = PyInt_AS_LONG(v);
w = PyDict_GetItem(scope, name);
assert(w && PyInt_Check(w));
i = PyInt_AS_LONG(w);
flags |= (i << SCOPE_OFF);
u = PyInt_FromLong(flags);
if (!u)
v_scope = PyDict_GetItem(scopes, name);
assert(v_scope && PyInt_Check(v_scope));
scope = PyInt_AS_LONG(v_scope);
flags |= (scope << SCOPE_OFFSET);
v_new = PyInt_FromLong(flags);
if (!v_new)
return 0;
if (PyDict_SetItem(symbols, name, u) < 0) {
Py_DECREF(u);
if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(v_new);
return 0;
}
Py_DECREF(u);
Py_DECREF(v_new);
}
free_value = PyInt_FromLong(FREE << SCOPE_OFF);
if (!free_value)
/* Record not yet resolved free variables from children (if any) */
v_free = PyInt_FromLong(FREE << SCOPE_OFFSET);
if (!v_free)
return 0;
/* add a free variable when it's only use is for creating a closure */
pos = 0;
while (PyDict_Next(free, &pos, &name, &v)) {
PyObject *o = PyDict_GetItem(symbols, name);
itr = PyObject_GetIter(free);
if (!itr)
goto error;
if (o) {
/* It could be a free variable in a method of
while ((name = PyIter_Next(itr))) {
v = PyDict_GetItem(symbols, name);
/* Handle symbol that already exists in this scope */
if (v) {
/* Handle a free variable in a method of
the class that has the same name as a local
or global in the class scope.
*/
if (classflag &&
PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) {
long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS;
o = PyInt_FromLong(i);
if (!o) {
Py_DECREF(free_value);
return 0;
PyInt_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
long flags = PyInt_AS_LONG(v) | DEF_FREE_CLASS;
v_new = PyInt_FromLong(flags);
if (!v_new) {
goto error;
}
if (PyDict_SetItem(symbols, name, o) < 0) {
Py_DECREF(o);
Py_DECREF(free_value);
return 0;
if (PyDict_SetItem(symbols, name, v_new) < 0) {
Py_DECREF(v_new);
goto error;
}
Py_DECREF(o);
Py_DECREF(v_new);
}
/* else it's not free, probably a cell */
/* It's a cell, or already a free variable in this scope */
Py_DECREF(name);
continue;
}
if (!PyDict_GetItem(bound, name))
/* Handle global symbol */
if (!PySet_Contains(bound, name)) {
Py_DECREF(name);
continue; /* it's a global */
if (PyDict_SetItem(symbols, name, free_value) < 0) {
Py_DECREF(free_value);
return 0;
}
/* Propagate new free symbol up the lexical stack */
if (PyDict_SetItem(symbols, name, v_free) < 0) {
goto error;
}
Py_DECREF(name);
}
Py_DECREF(free_value);
Py_DECREF(itr);
Py_DECREF(v_free);
return 1;
error:
Py_XDECREF(v_free);
Py_XDECREF(itr);
Py_XDECREF(name);
return 0;
}
/* Make final symbol table decisions for block of ste.
@ -611,59 +630,74 @@ static int
analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
PyObject *global)
{
PyObject *name, *v, *local = NULL, *scope = NULL, *newbound = NULL;
PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
PyObject *newglobal = NULL, *newfree = NULL;
int i, success = 0;
Py_ssize_t pos = 0;
local = PyDict_New();
scopes = PyDict_New();
if (!scopes)
goto error;
local = PySet_New(NULL);
if (!local)
goto error;
scope = PyDict_New();
if (!scope)
goto error;
newglobal = PyDict_New();
newglobal = PySet_New(NULL);
if (!newglobal)
goto error;
newfree = PyDict_New();
newfree = PySet_New(NULL);
if (!newfree)
goto error;
newbound = PyDict_New();
newbound = PySet_New(NULL);
if (!newbound)
goto error;
/* Class namespace has no effect on names visible in
nested functions, so populate the global and bound
sets to be passed to child blocks before analyzing
this one.
*/
if (ste->ste_type == ClassBlock) {
/* make a copy of globals before calling analyze_name(),
because global statements in the class have no effect
on nested functions.
*/
if (PyDict_Update(newglobal, global) < 0)
goto error;
if (bound)
if (PyDict_Update(newbound, bound) < 0)
/* Pass down previously bound symbols */
if (bound) {
if (!PyNumber_InPlaceOr(newbound, bound))
goto error;
Py_DECREF(newbound);
}
/* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error;
Py_DECREF(newglobal);
}
/* Analyze symbols in current scope */
assert(PySTEntry_Check(ste));
assert(PyDict_Check(ste->ste_symbols));
while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
long flags = PyInt_AS_LONG(v);
if (!analyze_name(ste, scope, name, flags, bound, local, free,
if (!analyze_name(ste, scopes, name, flags, bound, local, free,
global))
goto error;
}
/* Populate global and bound sets to be passed to children.
*/
if (ste->ste_type != ClassBlock) {
/* Add function locals to bound set */
if (ste->ste_type == FunctionBlock) {
if (PyDict_Update(newbound, local) < 0)
if (!PyNumber_InPlaceOr(newbound, local))
goto error;
Py_DECREF(newbound);
}
/* Pass down previously bound symbols */
if (bound) {
if (PyDict_Update(newbound, bound) < 0)
if (!PyNumber_InPlaceOr(newbound, bound))
goto error;
Py_DECREF(newbound);
}
if (PyDict_Update(newglobal, global) < 0)
/* Pass down known globals */
if (!PyNumber_InPlaceOr(newglobal, global))
goto error;
Py_DECREF(newglobal);
}
/* Recursively call analyze_block() on each child block */
@ -674,24 +708,28 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
entry = (PySTEntryObject*)c;
if (!analyze_block(entry, newbound, newfree, newglobal))
goto error;
/* Check if any children have free variables */
if (entry->ste_free || entry->ste_child_free)
ste->ste_child_free = 1;
}
if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree))
/* Check if any local variables need to be converted to cell variables */
if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
goto error;
if (!update_symbols(ste->ste_symbols, scope, bound, newfree,
/* Records the results of the analysis in the symbol table entry */
if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
ste->ste_type == ClassBlock))
goto error;
if (!check_unoptimized(ste))
goto error;
if (PyDict_Update(free, newfree) < 0)
if (!PyNumber_InPlaceOr(free, newfree))
goto error;
Py_DECREF(free);
success = 1;
error:
Py_XDECREF(scopes);
Py_XDECREF(local);
Py_XDECREF(scope);
Py_XDECREF(newbound);
Py_XDECREF(newglobal);
Py_XDECREF(newfree);
@ -706,10 +744,10 @@ symtable_analyze(struct symtable *st)
PyObject *free, *global;
int r;
free = PyDict_New();
free = PySet_New(NULL);
if (!free)
return 0;
global = PyDict_New();
global = PySet_New(NULL);
if (!global) {
Py_DECREF(free);
return 0;
@ -1200,16 +1238,18 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
case Set_kind:
VISIT_SEQ(st, expr, e->v.Set.elts);
break;
case ListComp_kind:
if (!symtable_new_tmpname(st))
return 0;
VISIT(st, expr, e->v.ListComp.elt);
VISIT_SEQ(st, comprehension, e->v.ListComp.generators);
break;
case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e))
return 0;
break;
case ListComp_kind:
if (!symtable_visit_listcomp(st, e))
return 0;
break;
case SetComp_kind:
if (!symtable_visit_setcomp(st, e))
return 0;
break;
case Yield_kind:
if (e->v.Yield.value)
VISIT(st, expr, e->v.Yield.value);
@ -1479,27 +1519,60 @@ symtable_visit_slice(struct symtable *st, slice_ty s)
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
symtable_handle_comprehension(struct symtable *st, expr_ty e,
identifier scope_name,
asdl_seq *generators, expr_ty elt)
{
int is_generator = (e->kind == GeneratorExp_kind);
int needs_tmp = !is_generator;
comprehension_ty outermost = ((comprehension_ty)
(asdl_seq_GET(e->v.GeneratorExp.generators, 0)));
asdl_seq_GET(generators, 0));
/* Outermost iterator is evaluated in current scope */
VISIT(st, expr, outermost->iter);
/* Create generator scope for the rest */
if (!GET_IDENTIFIER(genexpr) ||
!symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, 0)) {
/* Create comprehension scope for the rest */
if (!scope_name ||
!symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
return 0;
}
st->st_cur->ste_generator = 1;
st->st_cur->ste_generator = is_generator;
/* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) {
symtable_exit_block(st, (void *)e);
return 0;
}
/* Allocate temporary name if needed */
if (needs_tmp && !symtable_new_tmpname(st)) {
symtable_exit_block(st, (void *)e);
return 0;
}
VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e);
VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e);
VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension,
e->v.GeneratorExp.generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e);
generators, 1, (void*)e);
VISIT_IN_BLOCK(st, expr, elt, (void*)e);
return symtable_exit_block(st, (void *)e);
}
static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt);
}
static int
symtable_visit_listcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp),
e->v.ListComp.generators,
e->v.ListComp.elt);
}
static int
symtable_visit_setcomp(struct symtable *st, expr_ty e)
{
return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
e->v.SetComp.generators,
e->v.SetComp.elt);
}

4
setup.py
View File

@ -1175,8 +1175,8 @@ class PyBuildExt(build_ext):
#
include_dirs = [
join(F, fw + '.framework', H)
for fw in 'Tcl', 'Tk'
for H in 'Headers', 'Versions/Current/PrivateHeaders'
for fw in ('Tcl', 'Tk')
for H in ('Headers', 'Versions/Current/PrivateHeaders')
]
# For 8.4a2, the X11 headers are not included. Rather than include a