Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

PEP 3107 - Function Annotations thanks to Tony Lownds

This commit is contained in:
Neal Norwitz 2006-12-28 06:47:50 +00:00
parent f6657e67b3
commit c150536b5e
32 changed files with 2855 additions and 1897 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Grammar/Grammar
View File

@ -21,13 +21,20 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+ decorators: decorator+
funcdef: [decorators] 'def' NAME parameters ':' suite funcdef: [decorators] 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [varargslist] ')' parameters: '(' [typedargslist] ')'
varargslist: ((fpdef ['=' test] ',')* typedargslist: ((tfpdef ['=' test] ',')*
('*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME) | ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
fpdef ['=' test] (',' fpdef ['=' test])* [',']) | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
fpdef: NAME | '(' fplist ')' tname: NAME [':' test]
fplist: fpdef (',' fpdef)* [','] tfpdef: tname | '(' tfplist ')'
tfplist: tfpdef (',' tfpdef)* [',']
varargslist: ((vfpdef ['=' test] ',')*
('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname)
| vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
vname: NAME
vfpdef: vname | '(' vfplist ')'
vfplist: vfpdef (',' vfpdef)* [',']
stmt: simple_stmt | compound_stmt stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE

32
Include/Python-ast.h
View File

@ -30,6 +30,8 @@ typedef struct _excepthandler *excepthandler_ty;
typedef struct _arguments *arguments_ty; typedef struct _arguments *arguments_ty;
typedef struct _arg *arg_ty;
typedef struct _keyword *keyword_ty; typedef struct _keyword *keyword_ty;
typedef struct _alias *alias_ty; typedef struct _alias *alias_ty;
@ -74,6 +76,7 @@ struct _stmt {
arguments_ty args; arguments_ty args;
asdl_seq *body; asdl_seq *body;
asdl_seq *decorators; asdl_seq *decorators;
expr_ty returns;
} FunctionDef; } FunctionDef;
struct { struct {
@ -328,12 +331,30 @@ struct _excepthandler {
struct _arguments { struct _arguments {
asdl_seq *args; asdl_seq *args;
identifier vararg; identifier vararg;
expr_ty varargannotation;
asdl_seq *kwonlyargs; asdl_seq *kwonlyargs;
identifier kwarg; identifier kwarg;
expr_ty kwargannotation;
asdl_seq *defaults; asdl_seq *defaults;
asdl_seq *kw_defaults; asdl_seq *kw_defaults;
}; };
enum _arg_kind {SimpleArg_kind=1, NestedArgs_kind=2};
struct _arg {
enum _arg_kind kind;
union {
struct {
identifier arg;
expr_ty annotation;
} SimpleArg;
struct {
asdl_seq *args;
} NestedArgs;
} v;
};
struct _keyword { struct _keyword {
identifier arg; identifier arg;
expr_ty value; expr_ty value;
@ -350,8 +371,8 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena);
mod_ty Expression(expr_ty body, PyArena *arena); mod_ty Expression(expr_ty body, PyArena *arena);
mod_ty Suite(asdl_seq * body, PyArena *arena); mod_ty Suite(asdl_seq * body, PyArena *arena);
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorators, int lineno, int col_offset, PyArena asdl_seq * decorators, expr_ty returns, int lineno, int
*arena); col_offset, PyArena *arena);
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
lineno, int col_offset, PyArena *arena); lineno, int col_offset, PyArena *arena);
stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena); stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
@ -429,9 +450,12 @@ comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs,
PyArena *arena); PyArena *arena);
excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body, int excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body, int
lineno, int col_offset, PyArena *arena); lineno, int col_offset, PyArena *arena);
arguments_ty arguments(asdl_seq * args, identifier vararg, asdl_seq * arguments_ty arguments(asdl_seq * args, identifier vararg, expr_ty
kwonlyargs, identifier kwarg, asdl_seq * defaults, varargannotation, asdl_seq * kwonlyargs, identifier
kwarg, expr_ty kwargannotation, asdl_seq * defaults,
asdl_seq * kw_defaults, PyArena *arena); asdl_seq * kw_defaults, PyArena *arena);
arg_ty SimpleArg(identifier arg, expr_ty annotation, PyArena *arena);
arg_ty NestedArgs(asdl_seq * args, PyArena *arena);
keyword_ty keyword(identifier arg, expr_ty value, PyArena *arena); keyword_ty keyword(identifier arg, expr_ty value, PyArena *arena);
alias_ty alias(identifier name, identifier asname, PyArena *arena); alias_ty alias(identifier name, identifier asname, PyArena *arena);

5
Include/funcobject.h
View File

@ -30,6 +30,7 @@ typedef struct {
PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */
PyObject *func_weakreflist; /* List of weak references */ PyObject *func_weakreflist; /* List of weak references */
PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_module; /* The __module__ attribute, can be anything */
PyObject *func_annotations; /* Annotations, a dict or NULL */
/* Invariant: /* Invariant:
* func_closure contains the bindings for func_code->co_freevars, so * func_closure contains the bindings for func_code->co_freevars, so
@ -52,6 +53,8 @@ PyAPI_FUNC(PyObject *) PyFunction_GetKwDefaults(PyObject *);
PyAPI_FUNC(int) PyFunction_SetKwDefaults(PyObject *, PyObject *); PyAPI_FUNC(int) PyFunction_SetKwDefaults(PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyFunction_GetClosure(PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetClosure(PyObject *);
PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *);
PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *);
/* Macros for direct access to these values. Type checks are *not* /* Macros for direct access to these values. Type checks are *not*
done, so use with care. */ done, so use with care. */
@ -67,6 +70,8 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *);
(((PyFunctionObject *)func) -> func_kwdefaults) (((PyFunctionObject *)func) -> func_kwdefaults)
#define PyFunction_GET_CLOSURE(func) \ #define PyFunction_GET_CLOSURE(func) \
(((PyFunctionObject *)func) -> func_closure) (((PyFunctionObject *)func) -> func_closure)
#define PyFunction_GET_ANNOTATIONS(func) \
(((PyFunctionObject *)func) -> func_annotations)
/* The classmethod and staticmethod types lives here, too */ /* The classmethod and staticmethod types lives here, too */
PyAPI_DATA(PyTypeObject) PyClassMethod_Type; PyAPI_DATA(PyTypeObject) PyClassMethod_Type;

157
Include/graminit.h
View File

@ -5,79 +5,84 @@
#define decorators 260 #define decorators 260
#define funcdef 261 #define funcdef 261
#define parameters 262 #define parameters 262
#define varargslist 263 #define typedargslist 263
#define fpdef 264 #define tname 264
#define fplist 265 #define tfpdef 265
#define stmt 266 #define tfplist 266
#define simple_stmt 267 #define varargslist 267
#define small_stmt 268 #define vname 268
#define expr_stmt 269 #define vfpdef 269
#define augassign 270 #define vfplist 270
#define print_stmt 271 #define stmt 271
#define del_stmt 272 #define simple_stmt 272
#define pass_stmt 273 #define small_stmt 273
#define flow_stmt 274 #define expr_stmt 274
#define break_stmt 275 #define augassign 275
#define continue_stmt 276 #define print_stmt 276
#define return_stmt 277 #define del_stmt 277
#define yield_stmt 278 #define pass_stmt 278
#define raise_stmt 279 #define flow_stmt 279
#define import_stmt 280 #define break_stmt 280
#define import_name 281 #define continue_stmt 281
#define import_from 282 #define return_stmt 282
#define import_as_name 283 #define yield_stmt 283
#define dotted_as_name 284 #define raise_stmt 284
#define import_as_names 285 #define import_stmt 285
#define dotted_as_names 286 #define import_name 286
#define dotted_name 287 #define import_from 287
#define global_stmt 288 #define import_as_name 288
#define assert_stmt 289 #define dotted_as_name 289
#define compound_stmt 290 #define import_as_names 290
#define if_stmt 291 #define dotted_as_names 291
#define while_stmt 292 #define dotted_name 292
#define for_stmt 293 #define global_stmt 293
#define try_stmt 294 #define assert_stmt 294
#define with_stmt 295 #define compound_stmt 295
#define with_var 296 #define if_stmt 296
#define except_clause 297 #define while_stmt 297
#define suite 298 #define for_stmt 298
#define testlist_safe 299 #define try_stmt 299
#define old_test 300 #define with_stmt 300
#define old_lambdef 301 #define with_var 301
#define test 302 #define except_clause 302
#define or_test 303 #define suite 303
#define and_test 304 #define testlist_safe 304
#define not_test 305 #define old_test 305
#define comparison 306 #define old_lambdef 306
#define comp_op 307 #define test 307
#define expr 308 #define or_test 308
#define xor_expr 309 #define and_test 309
#define and_expr 310 #define not_test 310
#define shift_expr 311 #define comparison 311
#define arith_expr 312 #define comp_op 312
#define term 313 #define expr 313
#define factor 314 #define xor_expr 314
#define power 315 #define and_expr 315
#define atom 316 #define shift_expr 316
#define listmaker 317 #define arith_expr 317
#define testlist_gexp 318 #define term 318
#define lambdef 319 #define factor 319
#define trailer 320 #define power 320
#define subscriptlist 321 #define atom 321
#define subscript 322 #define listmaker 322
#define sliceop 323 #define testlist_gexp 323
#define exprlist 324 #define lambdef 324
#define testlist 325 #define trailer 325
#define dictsetmaker 326 #define subscriptlist 326
#define classdef 327 #define subscript 327
#define arglist 328 #define sliceop 328
#define argument 329 #define exprlist 329
#define list_iter 330 #define testlist 330
#define list_for 331 #define dictsetmaker 331
#define list_if 332 #define classdef 332
#define gen_iter 333 #define arglist 333
#define gen_for 334 #define argument 334
#define gen_if 335 #define list_iter 335
#define testlist1 336 #define list_for 336
#define encoding_decl 337 #define list_if 337
#define yield_expr 338 #define gen_iter 338
#define gen_for 339
#define gen_if 340
#define testlist1 341
#define encoding_decl 342
#define yield_expr 343

7
Include/token.h
View File

@ -58,10 +58,11 @@ extern "C" {
#define DOUBLESLASH 48 #define DOUBLESLASH 48
#define DOUBLESLASHEQUAL 49 #define DOUBLESLASHEQUAL 49
#define AT 50 #define AT 50
#define RARROW 51
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */ /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
#define OP 51 #define OP 52
#define ERRORTOKEN 52 #define ERRORTOKEN 53
#define N_TOKENS 53 #define N_TOKENS 54
/* Special definitions for cooperation with parser */ /* Special definitions for cooperation with parser */

88
Lib/compiler/ast.py
View File

@ -33,7 +33,10 @@ class Node:
pass # implemented by subclasses pass # implemented by subclasses
class EmptyNode(Node): class EmptyNode(Node):
pass def getChildNodes(self):
return ()
def getChildren(self):
return ()
class Expression(Node): class Expression(Node):
# Expression is an artificial node class to support "eval" # Expression is an artificial node class to support "eval"
@ -487,12 +490,13 @@ class From(Node):
return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level)) return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level))
class Function(Node): class Function(Node):
def __init__(self, decorators, name, argnames, defaults, kwonlyargs, flags, doc, code, lineno=None): def __init__(self, decorators, name, arguments, defaults, kwonlyargs, returns, flags, doc, code, lineno=None):
self.decorators = decorators self.decorators = decorators
self.name = name self.name = name
self.argnames = argnames self.arguments = arguments
self.defaults = defaults self.defaults = defaults
self.kwonlyargs = kwonlyargs self.kwonlyargs = kwonlyargs
self.returns = returns
self.flags = flags self.flags = flags
self.doc = doc self.doc = doc
self.code = code self.code = code
@ -508,9 +512,10 @@ class Function(Node):
children = [] children = []
children.append(self.decorators) children.append(self.decorators)
children.append(self.name) children.append(self.name)
children.append(self.argnames) children.extend(flatten(self.arguments))
children.extend(flatten(self.defaults)) children.extend(flatten(self.defaults))
children.append(self.kwonlyargs) children.extend(flatten(self.kwonlyargs))
children.append(self.returns)
children.append(self.flags) children.append(self.flags)
children.append(self.doc) children.append(self.doc)
children.append(self.code) children.append(self.code)
@ -520,18 +525,22 @@ class Function(Node):
nodelist = [] nodelist = []
if self.decorators is not None: if self.decorators is not None:
nodelist.append(self.decorators) nodelist.append(self.decorators)
nodelist.extend(flatten_nodes(self.arguments))
nodelist.extend(flatten_nodes(self.defaults)) nodelist.extend(flatten_nodes(self.defaults))
nodelist.extend(flatten_nodes(self.kwonlyargs))
if self.returns is not None:
nodelist.append(self.returns)
nodelist.append(self.code) nodelist.append(self.code)
return tuple(nodelist) return tuple(nodelist)
def __repr__(self): def __repr__(self):
return "Function(%s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.doc), repr(self.code)) return "Function(%s, %s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.returns), repr(self.flags), repr(self.doc), repr(self.code))
class GenExpr(Node): class GenExpr(Node):
def __init__(self, code, lineno=None): def __init__(self, code, lineno=None):
self.code = code self.code = code
self.lineno = lineno self.lineno = lineno
self.argnames = ['.0'] self.arguments = [SimpleArg('.0', None)]
self.varargs = self.kwargs = None self.varargs = self.kwargs = None
self.kwonlyargs = () self.kwonlyargs = ()
@ -715,9 +724,24 @@ class Keyword(Node):
def __repr__(self): def __repr__(self):
return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr)) return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr))
class Kwarg(Node):
def __init__(self, arg, expr, lineno=None):
self.arg = arg
self.expr = expr
self.lineno = lineno
def getChildren(self):
return self.arg, self.expr
def getChildNodes(self):
return self.arg, self.expr
def __repr__(self):
return "Kwarg(%s, %s)" % (repr(self.arg), repr(self.expr))
class Lambda(Node): class Lambda(Node):
def __init__(self, argnames, defaults, kwonlyargs, flags, code, lineno=None): def __init__(self, arguments, defaults, kwonlyargs, flags, code, lineno=None):
self.argnames = argnames self.arguments = arguments
self.defaults = defaults self.defaults = defaults
self.kwonlyargs = kwonlyargs self.kwonlyargs = kwonlyargs
self.flags = flags self.flags = flags
@ -728,25 +752,28 @@ class Lambda(Node):
self.varargs = 1 self.varargs = 1
if flags & CO_VARKEYWORDS: if flags & CO_VARKEYWORDS:
self.kwargs = 1 self.kwargs = 1
self.returns = None
def getChildren(self): def getChildren(self):
children = [] children = []
children.append(self.argnames) children.extend(flatten(self.arguments))
children.extend(flatten(self.defaults)) children.extend(flatten(self.defaults))
children.append(self.kwonlyargs) children.extend(flatten(self.kwonlyargs))
children.append(self.flags) children.append(self.flags)
children.append(self.code) children.append(self.code)
return tuple(children) return tuple(children)
def getChildNodes(self): def getChildNodes(self):
nodelist = [] nodelist = []
nodelist.extend(flatten_nodes(self.arguments))
nodelist.extend(flatten_nodes(self.defaults)) nodelist.extend(flatten_nodes(self.defaults))
nodelist.extend(flatten_nodes(self.kwonlyargs))
nodelist.append(self.code) nodelist.append(self.code)
return tuple(nodelist) return tuple(nodelist)
def __repr__(self): def __repr__(self):
return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code)) return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code))
class LeftShift(Node): class LeftShift(Node):
def __init__(self, (left, right), lineno=None): def __init__(self, (left, right), lineno=None):
@ -897,6 +924,22 @@ class Name(Node):
def __repr__(self): def __repr__(self):
return "Name(%s)" % (repr(self.name),) return "Name(%s)" % (repr(self.name),)
class NestedArgs(Node):
def __init__(self, args, lineno=None):
self.args = args
self.lineno = lineno
def getChildren(self):
return tuple(flatten(self.args))
def getChildNodes(self):
nodelist = []
nodelist.extend(flatten_nodes(self.args))
return tuple(nodelist)
def __repr__(self):
return "NestedArgs(%s)" % (repr(self.args),)
class Not(Node): class Not(Node):
def __init__(self, expr, lineno=None): def __init__(self, expr, lineno=None):
self.expr = expr self.expr = expr
@ -1071,6 +1114,27 @@ class Set(Node):
def __repr__(self): def __repr__(self):
return "Set(%s)" % (repr(self.items),) return "Set(%s)" % (repr(self.items),)
class SimpleArg(Node):
def __init__(self, name, annotation, lineno=None):
self.name = name
self.annotation = annotation
self.lineno = lineno
def getChildren(self):
children = []
children.append(self.name)
children.append(self.annotation)
return tuple(children)
def getChildNodes(self):
nodelist = []
if self.annotation is not None:
nodelist.append(self.annotation)
return tuple(nodelist)
def __repr__(self):
return "SimpleArg(%s, %s)" % (repr(self.name), repr(self.annotation))
class Slice(Node): class Slice(Node):
def __init__(self, expr, flags, lower, upper, lineno=None): def __init__(self, expr, flags, lower, upper, lineno=None):
self.expr = expr self.expr = expr

44
Lib/compiler/pyassem.py
View File

@ -314,7 +314,7 @@ class PyFlowGraph(FlowGraph):
super_init = FlowGraph.__init__ super_init = FlowGraph.__init__
def __init__(self, name, filename, def __init__(self, name, filename,
args=(), kwonlyargs={}, optimized=0, klass=None): args=(), kwonlyargs=(), optimized=0, klass=None):
self.super_init() self.super_init()
self.name = name self.name = name
self.filename = filename self.filename = filename
@ -338,24 +338,40 @@ class PyFlowGraph(FlowGraph):
# The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
# kinds of variables. # kinds of variables.
self.closure = [] self.closure = []
self.varnames = list(args) or [] # The varnames list needs to be computed after flags have been set
for i in range(len(self.varnames)): self.varnames = []
self.stage = RAW
def computeVarnames(self):
# self.args is positional, vararg, kwarg, kwonly, unpacked. This
# order is due to the visit order in symbol module and could change.
# argcount is # len(self.args) - len(unpacked). We want
# self.varnames to be positional, kwonly, vararg, kwarg, unpacked
# and argcount to be len(positional).
# determine starting index of unpacked, kwonly, vararg
u = self.argcount # starting index of unpacked
k = u - len(self.kwonlyargs) # starting index of kwonly
v = k - self.checkFlag(CO_VARARGS) - self.checkFlag(CO_VARKEYWORDS)
vars = list(self.args)
self.varnames = vars[:v] + vars[k:u] + vars[v:k] + vars[u:]
self.argcount = v
# replace TupleArgs with calculated var name
for i in range(self.argcount):
var = self.varnames[i] var = self.varnames[i]
if isinstance(var, TupleArg): if isinstance(var, TupleArg):
self.varnames[i] = var.getName() self.varnames[i] = var.getName()
self.stage = RAW
def setDocstring(self, doc): def setDocstring(self, doc):
self.docstring = doc self.docstring = doc
def setFlag(self, flag): def setFlag(self, flag):
self.flags = self.flags | flag self.flags = self.flags | flag
if flag == CO_VARARGS:
self.argcount = self.argcount - 1
def checkFlag(self, flag): def checkFlag(self, flag):
if self.flags & flag: return (self.flags & flag) == flag
return 1
def setFreeVars(self, names): def setFreeVars(self, names):
self.freevars = list(names) self.freevars = list(names)
@ -366,6 +382,7 @@ class PyFlowGraph(FlowGraph):
def getCode(self): def getCode(self):
"""Get a Python code object""" """Get a Python code object"""
assert self.stage == RAW assert self.stage == RAW
self.computeVarnames()
self.computeStackDepth() self.computeStackDepth()
self.flattenGraph() self.flattenGraph()
assert self.stage == FLAT assert self.stage == FLAT
@ -575,6 +592,12 @@ class PyFlowGraph(FlowGraph):
lnotab.nextLine(oparg) lnotab.nextLine(oparg)
continue continue
hi, lo = twobyte(oparg) hi, lo = twobyte(oparg)
extended, hi = twobyte(hi)
if extended:
ehi, elo = twobyte(extended)
lnotab.addCode(self.opnum['EXTENDED_ARG'], elo, ehi)
try: try:
lnotab.addCode(self.opnum[opname], lo, hi) lnotab.addCode(self.opnum[opname], lo, hi)
except ValueError: except ValueError:
@ -595,8 +618,6 @@ class PyFlowGraph(FlowGraph):
else: else:
nlocals = len(self.varnames) nlocals = len(self.varnames)
argcount = self.argcount argcount = self.argcount
if self.flags & CO_VARKEYWORDS:
argcount = argcount - 1
kwonlyargcount = len(self.kwonlyargs) kwonlyargcount = len(self.kwonlyargs)
return new.code(argcount, kwonlyargcount, return new.code(argcount, kwonlyargcount,
nlocals, self.stacksize, self.flags, nlocals, self.stacksize, self.flags,
@ -809,7 +830,8 @@ class StackDepthTracker:
return self.CALL_FUNCTION(argc)-2 return self.CALL_FUNCTION(argc)-2
def MAKE_FUNCTION(self, argc): def MAKE_FUNCTION(self, argc):
hi, lo = divmod(argc, 256) hi, lo = divmod(argc, 256)
return -(lo + hi * 2) ehi, hi = divmod(hi, 256)
return -(lo + hi * 2 + ehi)
def MAKE_CLOSURE(self, argc): def MAKE_CLOSURE(self, argc):
# XXX need to account for free variables too! # XXX need to account for free variables too!
return -argc return -argc

77
Lib/compiler/pycodegen.py
View File

@ -378,18 +378,57 @@ class CodeGenerator:
walk(node.code, gen) walk(node.code, gen)
gen.finish() gen.finish()
self.set_lineno(node) self.set_lineno(node)
num_kwargs = 0
for keyword in node.kwonlyargs: for keyword in node.kwonlyargs:
default = keyword.expr default = keyword.expr
if isinstance(default, ast.EmptyNode): if isinstance(default, ast.EmptyNode):
continue continue
self.emit('LOAD_CONST', keyword.name) self.emit('LOAD_CONST', keyword.arg.name)
self.visit(default) self.visit(default)
num_kwargs += 1
for default in node.defaults: for default in node.defaults:
self.visit(default) self.visit(default)
self._makeClosure(gen, len(node.defaults))
num_annotations = self._visit_annotations(node)
oparg = len(node.defaults)
oparg |= num_kwargs << 8
oparg |= num_annotations << 16
self._makeClosure(gen, oparg)
for i in range(ndecorators): for i in range(ndecorators):
self.emit('CALL_FUNCTION', 1) self.emit('CALL_FUNCTION', 1)
def _visit_annotations(self, node):
# emit code, return num_annotations
annotations = []
annotations.extend(self._visit_argument_annotations(node.arguments))
annotations.extend(self._visit_kwarg_annotations(node.kwonlyargs))
if node.returns:
self.visit(node.returns)
annotations.append('return')
if not annotations:
return 0
self.emit('LOAD_CONST', tuple(annotations))
return len(annotations) + 1
def _visit_argument_annotations(self, arguments):
for arg in arguments:
if isinstance(arg, ast.SimpleArg):
if arg.annotation:
self.visit(arg.annotation)
yield arg.name
else:
for name in self._visit_argument_annotations(arg.args):
yield name
def _visit_kwarg_annotations(self, kwargs):
for kwarg in kwargs:
arg = kwarg.arg
if arg.annotation:
self.visit(arg.annotation)
yield arg.name
def visitClass(self, node): def visitClass(self, node):
gen = self.ClassGen(node, self.scopes, gen = self.ClassGen(node, self.scopes,
self.get_module()) self.get_module())
@ -1323,7 +1362,7 @@ class AbstractFunctionCode:
else: else:
name = func.name name = func.name
args, hasTupleArg = generateArgList(func.argnames) args, hasTupleArg = generateArgList(func.arguments)
kwonlyargs = generateKwonlyArgList(func.kwonlyargs) kwonlyargs = generateKwonlyArgList(func.kwonlyargs)
self.graph = pyassem.PyFlowGraph(name, func.filename, args, self.graph = pyassem.PyFlowGraph(name, func.filename, args,
kwonlyargs=kwonlyargs, kwonlyargs=kwonlyargs,
@ -1334,7 +1373,7 @@ class AbstractFunctionCode:
if not isLambda and func.doc: if not isLambda and func.doc:
self.setDocstring(func.doc) self.setDocstring(func.doc)
lnf = walk(func.code, self.NameFinder(args), verbose=0) lnf = walk(func.code, self.NameFinder(args+kwonlyargs), verbose=0)
self.locals.push(lnf.getLocals()) self.locals.push(lnf.getLocals())
if func.varargs: if func.varargs:
self.graph.setFlag(CO_VARARGS) self.graph.setFlag(CO_VARARGS)
@ -1342,7 +1381,7 @@ class AbstractFunctionCode:
self.graph.setFlag(CO_VARKEYWORDS) self.graph.setFlag(CO_VARKEYWORDS)
self.set_lineno(func) self.set_lineno(func)
if hasTupleArg: if hasTupleArg:
self.generateArgUnpack(func.argnames) self.generateArgUnpack(func.arguments)
def get_module(self): def get_module(self):
return self.module return self.module
@ -1356,9 +1395,9 @@ class AbstractFunctionCode:
def generateArgUnpack(self, args): def generateArgUnpack(self, args):
for i in range(len(args)): for i in range(len(args)):
arg = args[i] arg = args[i]
if isinstance(arg, tuple): if isinstance(arg, ast.NestedArgs):
self.emit('LOAD_FAST', '.%d' % (i * 2)) self.emit('LOAD_FAST', '.%d' % (i * 2))
self.unpackSequence(arg) self.unpackSequence(tuple(_nested_names(arg)))
def unpackSequence(self, tup): def unpackSequence(self, tup):
if VERSION > 1: if VERSION > 1:
@ -1452,21 +1491,29 @@ def generateArgList(arglist):
count = 0 count = 0
for i in range(len(arglist)): for i in range(len(arglist)):
elt = arglist[i] elt = arglist[i]
if isinstance(elt, str): if isinstance(elt, ast.SimpleArg):
args.append(elt) args.append(elt.name)
elif isinstance(elt, tuple): elif isinstance(elt, ast.NestedArgs):
args.append(TupleArg(i * 2, elt)) t = tuple(_nested_names(elt))
extra.extend(misc.flatten(elt)) args.append(TupleArg(i * 2, t))
extra.extend(misc.flatten(t))
count = count + 1 count = count + 1
else: else:
raise ValueError, "unexpect argument type:", elt raise ValueError, "unexpect argument type:", elt
return args + extra, count return args + extra, count
def _nested_names(elt):
for arg in elt.args:
if isinstance(arg, ast.SimpleArg):
yield arg.name
elif isinstance(arg, ast.NestedArgs):
yield tuple(_nested_names(arg))
def generateKwonlyArgList(keywordOnlyArgs): def generateKwonlyArgList(keywordOnlyArgs):
kwonlyargs = {} kwonlyargs = []
for elt in keywordOnlyArgs: for elt in keywordOnlyArgs:
assert isinstance(elt, ast.Keyword) assert isinstance(elt, ast.Kwarg)
kwonlyargs[elt.name] = elt.expr kwonlyargs.append(elt.arg.name)
return kwonlyargs return kwonlyargs
def findOp(node): def findOp(node):

21
Lib/compiler/symbols.py
View File

@ -233,7 +233,12 @@ class SymbolVisitor:
if parent.nested or isinstance(parent, FunctionScope): if parent.nested or isinstance(parent, FunctionScope):
scope.nested = 1 scope.nested = 1
self.scopes[node] = scope self.scopes[node] = scope
self._do_args(scope, node.argnames)
args = node.arguments
for kwonly in node.kwonlyargs:
args.append(kwonly.arg)
self._do_arguments(scope, args)
self.visit(node.code, scope) self.visit(node.code, scope)
self.handle_free_vars(scope, parent) self.handle_free_vars(scope, parent)
@ -275,16 +280,18 @@ class SymbolVisitor:
if parent.nested or isinstance(parent, FunctionScope): if parent.nested or isinstance(parent, FunctionScope):
scope.nested = 1 scope.nested = 1
self.scopes[node] = scope self.scopes[node] = scope
self._do_args(scope, node.argnames) self._do_arguments(scope, node.arguments)
self.visit(node.code, scope) self.visit(node.code, scope)
self.handle_free_vars(scope, parent) self.handle_free_vars(scope, parent)
def _do_args(self, scope, args): def _do_arguments(self, scope, arguments):
for name in args: for node in arguments:
if type(name) == types.TupleType: if isinstance(node, ast.SimpleArg):
self._do_args(scope, name) scope.add_param(node.name)
if node.annotation:
self.visit(node.annotation, scope)
else: else:
scope.add_param(name) self._do_arguments(scope, node.args)
def handle_free_vars(self, scope, parent): def handle_free_vars(self, scope, parent):
parent.add_child(scope) parent.add_child(scope)

118
Lib/compiler/transformer.py
View File

@ -234,25 +234,24 @@ class Transformer:
return Decorators(items) return Decorators(items)
def funcdef(self, nodelist): def funcdef(self, nodelist):
# -6 -5 -4 -3 -2 -1 # 0 1 2 4 -1
# funcdef: [decorators] 'def' NAME parameters ':' suite # funcdef: [decorators] 'def' NAME parameters ['->' test] ':' suite
# parameters: '(' [varargslist] ')' # parameters: '(' [typedargslist] ')'
if nodelist[0][0] == symbol.decorators:
if len(nodelist) == 6:
assert nodelist[0][0] == symbol.decorators
decorators = self.decorators(nodelist[0][1:]) decorators = self.decorators(nodelist[0][1:])
nodelist = nodelist[1:]
else: else:
assert len(nodelist) == 5
decorators = None decorators = None
assert len(nodelist) in (5, 7)
lineno = nodelist[-4][2] lineno = nodelist[0][2]
name = nodelist[-4][1] name = nodelist[1][1]
args = nodelist[-3][2] args = nodelist[2][2]
if args[0] == symbol.varargslist: if args[0] == symbol.varargslist or args[0] == symbol.typedargslist:
names, defaults, kwonlyargs, flags = self.com_arglist(args[1:]) arguments, defaults, kwonly, flags = self.com_arglist(args[1:])
else: else:
names = defaults = kwonlyargs = () arguments = defaults = kwonly = ()
flags = 0 flags = 0
doc = self.get_docstring(nodelist[-1]) doc = self.get_docstring(nodelist[-1])
@ -263,22 +262,28 @@ class Transformer:
assert isinstance(code, Stmt) assert isinstance(code, Stmt)
assert isinstance(code.nodes[0], Discard) assert isinstance(code.nodes[0], Discard)
del code.nodes[0] del code.nodes[0]
return Function(decorators, name, names, defaults,
kwonlyargs, flags, doc, code, lineno=lineno) if len(nodelist) == 7:
returns = self.com_node(nodelist[4])
else:
returns = None
return Function(decorators, name, arguments, defaults,
kwonly, returns, flags, doc, code, lineno=lineno)
def lambdef(self, nodelist): def lambdef(self, nodelist):
# lambdef: 'lambda' [varargslist] ':' test # lambdef: 'lambda' [varargslist] ':' test
if nodelist[2][0] == symbol.varargslist: if nodelist[2][0] == symbol.varargslist:
names, defaults, kwonlyargs, flags = \ arguments, defaults, kwonlyargs, flags = \
self.com_arglist(nodelist[2][1:]) self.com_arglist(nodelist[2][1:])
else: else:
names = defaults = kwonlyargs = () arguments = defaults = kwonlyargs = ()
flags = 0 flags = 0
# code for lambda # code for lambda
code = self.com_node(nodelist[-1]) code = self.com_node(nodelist[-1])
return Lambda(names, defaults, kwonlyargs, return Lambda(arguments, defaults, kwonlyargs,
flags, code, lineno=nodelist[1][2]) flags, code, lineno=nodelist[1][2])
old_lambdef = lambdef old_lambdef = lambdef
@ -324,10 +329,25 @@ class Transformer:
def varargslist(self, nodelist): def varargslist(self, nodelist):
raise WalkerError raise WalkerError
def fpdef(self, nodelist): def vfpdef(self, nodelist):
raise WalkerError raise WalkerError
def fplist(self, nodelist): def vfplist(self, nodelist):
raise WalkerError
def vname(self, nodelist):
raise WalkerError
def typedargslist(self, nodelist):
raise WalkerError
def tfpdef(self, nodelist):
raise WalkerError
def tfplist(self, nodelist):
raise WalkerError
def tname(self, nodelist):
raise WalkerError raise WalkerError
def dotted_name(self, nodelist): def dotted_name(self, nodelist):
@ -786,9 +806,10 @@ class Transformer:
return Discard(Const(None)) return Discard(Const(None))
def keywordonlyargs(self, nodelist): def keywordonlyargs(self, nodelist):
# (',' NAME ['=' test])* # (',' tname ['=' test])*
# ^^^ # ^^^
# ------+ # ------+
# tname and vname are handled.
kwonlyargs = [] kwonlyargs = []
i = 0 i = 0
while i < len(nodelist): while i < len(nodelist):
@ -802,10 +823,25 @@ class Transformer:
i += 2 i += 2
if node[0] == token.DOUBLESTAR: if node[0] == token.DOUBLESTAR:
return kwonlyargs, i return kwonlyargs, i
elif node[0] == token.NAME: elif node[0] in (symbol.vname, symbol.tname):
kwonlyargs.append(Keyword(node[1], default, lineno=node[2])) lineno = extractLineNo(node)
kwarg = Kwarg(self._simplearg(node), default, lineno=lineno)
kwonlyargs.append(kwarg)
i += 2 i += 2
return kwonlyargs, i return kwonlyargs, i
def _simplearg(self, node):
# tname: NAME [':' test]
# vname: NAME
assert node[0] == symbol.vname or node[0] == symbol.tname
name = node[1][1]
lineno = node[1][2]
assert isinstance(name, str)
if len(node) > 2:
annotation = self.com_node(node[3])
else:
annotation = None
return SimpleArg(name, annotation, lineno)
def com_arglist(self, nodelist): def com_arglist(self, nodelist):
# varargslist: # varargslist:
@ -814,7 +850,7 @@ class Transformer:
# | fpdef ['=' test] (',' fpdef ['=' test])* [','] # | fpdef ['=' test] (',' fpdef ['=' test])* [',']
# fpdef: NAME | '(' fplist ')' # fpdef: NAME | '(' fplist ')'
# fplist: fpdef (',' fpdef)* [','] # fplist: fpdef (',' fpdef)* [',']
names = [] arguments = []
kwonlyargs = [] kwonlyargs = []
defaults = [] defaults = []
flags = 0 flags = 0
@ -825,14 +861,15 @@ class Transformer:
if node[0] == token.STAR or node[0] == token.DOUBLESTAR: if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
if node[0] == token.STAR: if node[0] == token.STAR:
node = nodelist[i+1] node = nodelist[i+1]
if node[0] == token.NAME: # vararg if node[0] in (symbol.tname, symbol.vname): # vararg
names.append(node[1]) arguments.append(self._simplearg(node))
flags = flags | CO_VARARGS flags = flags | CO_VARARGS
i = i + 3 i = i + 3
else: # no vararg else: # no vararg
assert node[0] == token.COMMA assert node[0] == token.COMMA
i += 2 i += 2
if i < len(nodelist) and nodelist[i][0] == token.NAME: if i < len(nodelist) and \
nodelist[i][0] in (symbol.tname, symbol.vname):
kwonlyargs, skip = self.keywordonlyargs(nodelist[i:]) kwonlyargs, skip = self.keywordonlyargs(nodelist[i:])
i += skip i += skip
@ -843,13 +880,13 @@ class Transformer:
node = nodelist[i+1] node = nodelist[i+1]
else: else:
raise ValueError, "unexpected token: %s" % t raise ValueError, "unexpected token: %s" % t
names.append(node[1]) arguments.append(self._simplearg(node))
flags = flags | CO_VARKEYWORDS flags = flags | CO_VARKEYWORDS
break break
# fpdef: NAME | '(' fplist ')' # tfpdef: tname | '(' tfplist ')'
names.append(self.com_fpdef(node)) arguments.append(self.com_tfpdef(node))
i = i + 1 i = i + 1
if i < len(nodelist) and nodelist[i][0] == token.EQUAL: if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
@ -863,21 +900,24 @@ class Transformer:
# skip the comma # skip the comma
i = i + 1 i = i + 1
return names, defaults, kwonlyargs, flags return arguments, defaults, kwonlyargs, flags
def com_fpdef(self, node): def com_tfpdef(self, node):
# fpdef: NAME | '(' fplist ')' # tfpdef: tname | '(' tfplist ')'
# def f((x)): -- x is not nested
while node[1][0] == token.LPAR and len(node[2]) == 2:
node = node[2][1]
if node[1][0] == token.LPAR: if node[1][0] == token.LPAR:
return self.com_fplist(node[2]) return NestedArgs(self.com_tfplist(node[2]))
return node[1][1] return self._simplearg(node[1])
def com_fplist(self, node): def com_tfplist(self, node):
# fplist: fpdef (',' fpdef)* [','] # tfplist: tfpdef (',' tfpdef)* [',']
if len(node) == 2: if len(node) == 2:
return self.com_fpdef(node[1]) return self.com_tfpdef(node[1]),
list = [] list = []
for i in range(1, len(node), 2): for i in range(1, len(node), 2):
list.append(self.com_fpdef(node[i])) list.append(self.com_tfpdef(node[i]))
return tuple(list) return tuple(list)
def com_dotted_name(self, node): def com_dotted_name(self, node):

157
Lib/symbol.py
View File

@ -17,82 +17,87 @@ decorator = 259
decorators = 260 decorators = 260
funcdef = 261 funcdef = 261
parameters = 262 parameters = 262
varargslist = 263 typedargslist = 263
fpdef = 264 tname = 264
fplist = 265 tfpdef = 265
stmt = 266 tfplist = 266
simple_stmt = 267 varargslist = 267
small_stmt = 268 vname = 268
expr_stmt = 269 vfpdef = 269
augassign = 270 vfplist = 270
print_stmt = 271 stmt = 271
del_stmt = 272 simple_stmt = 272
pass_stmt = 273 small_stmt = 273
flow_stmt = 274 expr_stmt = 274
break_stmt = 275 augassign = 275
continue_stmt = 276 print_stmt = 276
return_stmt = 277 del_stmt = 277
yield_stmt = 278 pass_stmt = 278
raise_stmt = 279 flow_stmt = 279
import_stmt = 280 break_stmt = 280
import_name = 281 continue_stmt = 281
import_from = 282 return_stmt = 282
import_as_name = 283 yield_stmt = 283
dotted_as_name = 284 raise_stmt = 284
import_as_names = 285 import_stmt = 285
dotted_as_names = 286 import_name = 286
dotted_name = 287 import_from = 287
global_stmt = 288 import_as_name = 288
assert_stmt = 289 dotted_as_name = 289
compound_stmt = 290 import_as_names = 290
if_stmt = 291 dotted_as_names = 291
while_stmt = 292 dotted_name = 292
for_stmt = 293 global_stmt = 293
try_stmt = 294 assert_stmt = 294
with_stmt = 295 compound_stmt = 295
with_var = 296 if_stmt = 296
except_clause = 297 while_stmt = 297
suite = 298 for_stmt = 298
testlist_safe = 299 try_stmt = 299
old_test = 300 with_stmt = 300
old_lambdef = 301 with_var = 301
test = 302 except_clause = 302
or_test = 303 suite = 303
and_test = 304 testlist_safe = 304
not_test = 305 old_test = 305
comparison = 306 old_lambdef = 306
comp_op = 307 test = 307
expr = 308 or_test = 308
xor_expr = 309 and_test = 309
and_expr = 310 not_test = 310
shift_expr = 311 comparison = 311
arith_expr = 312 comp_op = 312
term = 313 expr = 313
factor = 314 xor_expr = 314
power = 315 and_expr = 315
atom = 316 shift_expr = 316
listmaker = 317 arith_expr = 317
testlist_gexp = 318 term = 318
lambdef = 319 factor = 319
trailer = 320 power = 320
subscriptlist = 321 atom = 321
subscript = 322 listmaker = 322
sliceop = 323 testlist_gexp = 323
exprlist = 324 lambdef = 324
testlist = 325 trailer = 325
dictsetmaker = 326 subscriptlist = 326
classdef = 327 subscript = 327
arglist = 328 sliceop = 328
argument = 329 exprlist = 329
list_iter = 330 testlist = 330
list_for = 331 dictsetmaker = 331
list_if = 332 classdef = 332
gen_iter = 333 arglist = 333
gen_for = 334 argument = 334
gen_if = 335 list_iter = 335
testlist1 = 336 list_for = 336
encoding_decl = 337 list_if = 337
yield_expr = 338 gen_iter = 338
gen_for = 339
gen_if = 340
testlist1 = 341
encoding_decl = 342
yield_expr = 343
#--end constants-- #--end constants--
sym_name = {} sym_name = {}

18
Lib/test/output/test_tokenize
View File

@ -682,4 +682,20 @@ test_tokenize
177,11-177,15: NAME 'pass' 177,11-177,15: NAME 'pass'
177,15-177,16: NEWLINE '\n' 177,15-177,16: NEWLINE '\n'
178,0-178,1: NL '\n' 178,0-178,1: NL '\n'
179,0-179,0: ENDMARKER '' 179,0-179,1: OP '@'
179,1-179,13: NAME 'staticmethod'
179,13-179,14: NEWLINE '\n'
180,0-180,3: NAME 'def'
180,4-180,7: NAME 'foo'
180,7-180,8: OP '('
180,8-180,9: NAME 'x'
180,9-180,10: OP ':'
180,10-180,11: NUMBER '1'
180,11-180,12: OP ')'
180,12-180,14: OP '->'
180,14-180,15: NUMBER '1'
180,15-180,16: OP ':'
180,17-180,21: NAME 'pass'
180,21-180,22: NEWLINE '\n'
181,0-181,1: NL '\n'
182,0-182,0: ENDMARKER ''

8
Lib/test/test_ast.py
View File

@ -151,9 +151,9 @@ def run_tests():
#### EVERYTHING BELOW IS GENERATED ##### #### EVERYTHING BELOW IS GENERATED #####
exec_results = [ exec_results = [
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Pass', (1, 9))], [])]), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]), ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]), ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]), ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]), ('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
@ -180,13 +180,13 @@ eval_results = [
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])), ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))), ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))), ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], None, [], []), ('Name', (1, 7), 'None', ('Load',)))), ('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])), ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])), ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), ('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
('Expression', ('Num', (1, 0), 10L)), ('Expression', ('Num', (1, 0), 10)),
('Expression', ('Str', (1, 0), 'string')), ('Expression', ('Str', (1, 0), 'string')),
('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),

41
Lib/test/test_compiler.py
View File

@ -115,6 +115,24 @@ class CompilerTest(unittest.TestCase):
dct = {} dct = {}
exec(c, dct) exec(c, dct)
self.assertEquals(dct.get('result'), 3) self.assertEquals(dct.get('result'), 3)
c = compiler.compile('def g(a):\n'
' def f(): return a + 2\n'
' return f()\n'
'result = g(1)',
'<string>',
'exec')
dct = {}
exec(c, dct)
self.assertEquals(dct.get('result'), 3)
c = compiler.compile('def g((a, b)):\n'
' def f(): return a + b\n'
' return f()\n'
'result = g((1, 2))',
'<string>',
'exec')
dct = {}
exec(c, dct)
self.assertEquals(dct.get('result'), 3)
def testGenExp(self): def testGenExp(self):
c = compiler.compile('list((i,j) for i in range(3) if i < 3' c = compiler.compile('list((i,j) for i in range(3) if i < 3'
@ -123,6 +141,22 @@ class CompilerTest(unittest.TestCase):
'eval') 'eval')
self.assertEquals(eval(c), [(0, 3), (1, 3), (2, 3)]) self.assertEquals(eval(c), [(0, 3), (1, 3), (2, 3)])
def testFuncAnnotations(self):
testdata = [
('def f(a: 1): pass', {'a': 1}),
('''def f(a, (b:1, c:2, d), e:3=4, f=5,
*g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass
''', {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
'k': 11, 'return': 12}),
]
for sourcecode, expected in testdata:
# avoid IndentationError: unexpected indent from trailing lines
sourcecode = sourcecode.rstrip()+'\n'
c = compiler.compile(sourcecode, '<string>', 'exec')
dct = {}
exec(c, dct)
self.assertEquals(dct['f'].func_annotations, expected)
NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard) NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
@ -167,10 +201,11 @@ from math import *
############################################################################### ###############################################################################
def test_main(): def test_main(all=False):
global TEST_ALL global TEST_ALL
TEST_ALL = test.test_support.is_resource_enabled("compiler") TEST_ALL = all or test.test_support.is_resource_enabled("compiler")
test.test_support.run_unittest(CompilerTest) test.test_support.run_unittest(CompilerTest)
if __name__ == "__main__": if __name__ == "__main__":
test_main() import sys
test_main('all' in sys.argv)

48
Lib/test/test_grammar.py
View File

@ -138,16 +138,22 @@ class GrammarTests(unittest.TestCase):
x = eval('1, 0 or 1') x = eval('1, 0 or 1')
def testFuncdef(self): def testFuncdef(self):
### 'def' NAME parameters ':' suite ### [decorators] 'def' NAME parameters ['->' test] ':' suite
### parameters: '(' [varargslist] ')' ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
### varargslist: (fpdef ['=' test] ',')* ### decorators: decorator+
### ('*' (NAME|',' fpdef ['=' test]) [',' ('**'|'*' '*') NAME] ### parameters: '(' [typedargslist] ')'
### | ('**'|'*' '*') NAME) ### typedargslist: ((tfpdef ['=' test] ',')*
### | fpdef ['=' test] (',' fpdef ['=' test])* [','] ### ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
### fpdef: NAME | '(' fplist ')' ### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
### fplist: fpdef (',' fpdef)* [','] ### tname: NAME [':' test]
### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test) ### tfpdef: tname | '(' tfplist ')'
### argument: [test '='] test # Really [keyword '='] test ### tfplist: tfpdef (',' tfpdef)* [',']
### varargslist: ((vfpdef ['=' test] ',')*
### ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname)
### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
### vname: NAME
### vfpdef: vname | '(' vfplist ')'
### vfplist: vfpdef (',' vfpdef)* [',']
def f1(): pass def f1(): pass
f1() f1()
f1(*()) f1(*())
@ -294,6 +300,28 @@ class GrammarTests(unittest.TestCase):
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
# argument annotation tests
def f(x) -> list: pass
self.assertEquals(f.func_annotations, {'return': list})
def f(x:int): pass
self.assertEquals(f.func_annotations, {'x': int})
def f(*x:str): pass
self.assertEquals(f.func_annotations, {'x': str})
def f(**x:float): pass
self.assertEquals(f.func_annotations, {'x': float})
def f(x, y:1+2): pass
self.assertEquals(f.func_annotations, {'y': 3})
def f(a, (b:1, c:2, d)): pass
self.assertEquals(f.func_annotations, {'b': 1, 'c': 2})
def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass
self.assertEquals(f.func_annotations,
{'b': 1, 'c': 2, 'e': 3, 'g': 6})
def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
**k:11) -> 12: pass
self.assertEquals(f.func_annotations,
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
'k': 11, 'return': 12})
def testLambdef(self): def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test ### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0 l1 = lambda : 0

10
Lib/test/test_tokenize.py
View File

@ -219,5 +219,15 @@ def foo():
if verbose: if verbose:
print 'finished' print 'finished'
def test_rarrow():
"""
This function exists solely to test the tokenization of the RARROW
operator.
>>> tokenize(iter(['->']).next) #doctest: +NORMALIZE_WHITESPACE
1,0-1,2:\tOP\t'->'
2,0-2,0:\tENDMARKER\t''
"""
if __name__ == "__main__": if __name__ == "__main__":
test_main() test_main()

3
Lib/test/tokenize_tests.txt
View File

@ -176,3 +176,6 @@ x = sys.modules['time'].time()
@staticmethod @staticmethod
def foo(): pass def foo(): pass
@staticmethod
def foo(x:1)->1: pass

7
Lib/token.py
View File

@ -60,9 +60,10 @@ DOUBLESTAREQUAL = 47
DOUBLESLASH = 48 DOUBLESLASH = 48
DOUBLESLASHEQUAL = 49 DOUBLESLASHEQUAL = 49
AT = 50 AT = 50
OP = 51 RARROW = 51
ERRORTOKEN = 52 OP = 52
N_TOKENS = 53 ERRORTOKEN = 53
N_TOKENS = 54
NT_OFFSET = 256 NT_OFFSET = 256
#--end constants-- #--end constants--

2
Lib/tokenize.py
View File

@ -78,7 +78,7 @@ String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
# longest operators first (e.g., if = came before ==, == would get # longest operators first (e.g., if = came before ==, == would get
# recognized as two instances of =). # recognized as two instances of =).
Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
r"//=?", r"//=?", r"->",
r"[+\-*/%&|^=<>]=?", r"[+\-*/%&|^=<>]=?",
r"~") r"~")

2
Misc/NEWS
View File

@ -36,6 +36,8 @@ TO DO
Core and Builtins Core and Builtins
----------------- -----------------
- Added function annotations per PEP 3107.
- Moved intern() to sys.intern(). - Moved intern() to sys.intern().
- exec is now a function. - exec is now a function.

135
Modules/parsermodule.c
View File

@ -854,7 +854,7 @@ VALIDATER(node); VALIDATER(small_stmt);
VALIDATER(class); VALIDATER(node); VALIDATER(class); VALIDATER(node);
VALIDATER(parameters); VALIDATER(suite); VALIDATER(parameters); VALIDATER(suite);
VALIDATER(testlist); VALIDATER(varargslist); VALIDATER(testlist); VALIDATER(varargslist);
VALIDATER(fpdef); VALIDATER(fplist); VALIDATER(vfpdef); VALIDATER(vfplist);
VALIDATER(stmt); VALIDATER(simple_stmt); VALIDATER(stmt); VALIDATER(simple_stmt);
VALIDATER(expr_stmt); VALIDATER(power); VALIDATER(expr_stmt); VALIDATER(power);
VALIDATER(print_stmt); VALIDATER(del_stmt); VALIDATER(print_stmt); VALIDATER(del_stmt);
@ -863,7 +863,7 @@ VALIDATER(raise_stmt); VALIDATER(import_stmt);
VALIDATER(import_name); VALIDATER(import_from); VALIDATER(import_name); VALIDATER(import_from);
VALIDATER(global_stmt); VALIDATER(list_if); VALIDATER(global_stmt); VALIDATER(list_if);
VALIDATER(assert_stmt); VALIDATER(list_for); VALIDATER(assert_stmt); VALIDATER(list_for);
VALIDATER(compound_stmt); VALIDATER(compound_stmt); VALIDATER(vname);
VALIDATER(while); VALIDATER(for); VALIDATER(while); VALIDATER(for);
VALIDATER(try); VALIDATER(except_clause); VALIDATER(try); VALIDATER(except_clause);
VALIDATER(test); VALIDATER(and_test); VALIDATER(test); VALIDATER(and_test);
@ -1120,7 +1120,32 @@ validate_testlist_safe(node *tree)
} }
/* '*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME /* validate either vname or tname.
* vname: NAME
* tname: NAME [':' test]
*/
static int
validate_vname(node *tree)
{
int nch = NCH(tree);
if (TYPE(tree) == vname) {
return nch == 1 && validate_name(CHILD(tree, 0), NULL);
}
else if (TYPE(tree) == tname) {
if (nch == 1) {
return validate_name(CHILD(tree, 0), NULL);
}
else if (nch == 3) {
return validate_name(CHILD(tree, 0), NULL) &&
validate_colon(CHILD(tree, 1)) &&
validate_test(CHILD(tree, 2));
}
}
return 0;
}
/* '*' vname (',' vname ['=' test])* [',' '**' vname] | '**' vname
* ..or tname in place of vname. vname: NAME; tname: NAME [':' test]
*/ */
static int static int
validate_varargslist_trailer(node *tree, int start) validate_varargslist_trailer(node *tree, int start)
@ -1136,26 +1161,27 @@ validate_varargslist_trailer(node *tree, int start)
sym = TYPE(CHILD(tree, start)); sym = TYPE(CHILD(tree, start));
if (sym == STAR) { if (sym == STAR) {
/* /*
* '*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME * '*' vname (',' vname ['=' test])* [',' '**' vname] | '**' vname
*/ */
if (nch-start == 2) if (nch-start == 2)
res = validate_name(CHILD(tree, start+1), NULL); res = validate_vname(CHILD(tree, start+1));
else if (nch-start == 5 && TYPE(CHILD(tree, start+2)) == COMMA) else if (nch-start == 5 && TYPE(CHILD(tree, start+2)) == COMMA)
res = (validate_name(CHILD(tree, start+1), NULL) res = (validate_vname(CHILD(tree, start+1))
&& validate_comma(CHILD(tree, start+2)) && validate_comma(CHILD(tree, start+2))
&& validate_doublestar(CHILD(tree, start+3)) && validate_doublestar(CHILD(tree, start+3))
&& validate_name(CHILD(tree, start+4), NULL)); && validate_vname(CHILD(tree, start+4)));
else { else {
/* skip over [NAME] (',' NAME ['=' test])* */ /* skip over vname (',' vname ['=' test])* */
i = start + 1; i = start + 1;
if (TYPE(CHILD(tree, i)) == NAME) { /* skip over [NAME] */ if (TYPE(CHILD(tree, i)) == vname ||
TYPE(CHILD(tree, i)) == tname) { /* skip over vname or tname */
i += 1; i += 1;
} }
while (res && i+1 < nch) { /* validate (',' NAME ['=' test])* */ while (res && i+1 < nch) { /* validate (',' vname ['=' test])* */
res = validate_comma(CHILD(tree, i)); res = validate_comma(CHILD(tree, i));
if (TYPE(CHILD(tree, i+1)) == DOUBLESTAR) if (TYPE(CHILD(tree, i+1)) == DOUBLESTAR)
break; break;
res = res && validate_name(CHILD(tree, i+1), NULL); res = res && validate_vname(CHILD(tree, i+1));
if (res && i+2 < nch && TYPE(CHILD(tree, i+2)) == EQUAL) { if (res && i+2 < nch && TYPE(CHILD(tree, i+2)) == EQUAL) {
res = res && (i+3 < nch) res = res && (i+3 < nch)
&& validate_test(CHILD(tree, i+3)); && validate_test(CHILD(tree, i+3));
@ -1165,9 +1191,9 @@ validate_varargslist_trailer(node *tree, int start)
i += 2; i += 2;
} }
} }
/* [',' '**' NAME] */ /* [',' '**' vname] */
if (res && i+1 < nch && TYPE(CHILD(tree, i+1)) == DOUBLESTAR) { if (res && i+1 < nch && TYPE(CHILD(tree, i+1)) == DOUBLESTAR) {
res = validate_name(CHILD(tree, i+2), NULL); res = validate_vname(CHILD(tree, i+2));
} }
} }
} }
@ -1176,7 +1202,7 @@ validate_varargslist_trailer(node *tree, int start)
* '**' NAME * '**' NAME
*/ */
if (nch-start == 2) if (nch-start == 2)
res = validate_name(CHILD(tree, start+1), NULL); res = validate_vname(CHILD(tree, start+1));
} }
if (!res) if (!res)
err_string("illegal variable argument trailer for varargslist"); err_string("illegal variable argument trailer for varargslist");
@ -1184,21 +1210,34 @@ validate_varargslist_trailer(node *tree, int start)
} }
/* validate_varargslist() /* validate_varargslist()
* *
* varargslist: * Validate typedargslist or varargslist.
* (fpdef ['=' test] ',')* *
* ('*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME) * typedargslist: ((tfpdef ['=' test] ',')*
* | fpdef ['=' test] (',' fpdef ['=' test])* [','] * ('*' [tname] (',' tname ['=' test])* [',' '**' tname] |
* '**' tname)
* | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
* tname: NAME [':' test]
* tfpdef: tname | '(' tfplist ')'
* tfplist: tfpdef (',' tfpdef)* [',']
* varargslist: ((vfpdef ['=' test] ',')*
* ('*' [vname] (',' vname ['=' test])* [',' '**' vname] |
* '**' vname)
* | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
* vname: NAME
* vfpdef: vname | '(' vfplist ')'
* vfplist: vfpdef (',' vfpdef)* [',']
* *
*/ */
static int static int
validate_varargslist(node *tree) validate_varargslist(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res = validate_ntype(tree, varargslist) && (nch != 0); int res = (TYPE(tree) == varargslist ||
TYPE(tree) == typedargslist) &&
(nch != 0);
int sym; int sym;
if (!res) if (!res)
return 0; return 0;
if (nch < 1) { if (nch < 1) {
@ -1211,19 +1250,19 @@ validate_varargslist(node *tree)
* '*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME * '*' [NAME] (',' NAME ['=' test])* [',' '**' NAME] | '**' NAME
*/ */
res = validate_varargslist_trailer(tree, 0); res = validate_varargslist_trailer(tree, 0);
else if (sym == fpdef) { else if (sym == vfpdef || sym == tfpdef) {
int i = 0; int i = 0;
sym = TYPE(CHILD(tree, nch-1)); sym = TYPE(CHILD(tree, nch-1));
if (sym == NAME) { if (sym == vname || sym == tname) {
/* /*
* (fpdef ['=' test] ',')+ * (vfpdef ['=' test] ',')+
* ('*' NAME [',' '**' NAME] * ('*' vname [',' '**' vname]
* | '**' NAME) * | '**' vname)
*/ */
/* skip over (fpdef ['=' test] ',')+ */ /* skip over (vfpdef ['=' test] ',')+ */
while (res && (i+2 <= nch)) { while (res && (i+2 <= nch)) {
res = validate_fpdef(CHILD(tree, i)); res = validate_vfpdef(CHILD(tree, i));
++i; ++i;
if (res && TYPE(CHILD(tree, i)) == EQUAL && (i+2 <= nch)) { if (res && TYPE(CHILD(tree, i)) == EQUAL && (i+2 <= nch)) {
res = (validate_equal(CHILD(tree, i)) res = (validate_equal(CHILD(tree, i))
@ -1248,7 +1287,7 @@ validate_varargslist(node *tree)
} }
else { else {
/* /*
* fpdef ['=' test] (',' fpdef ['=' test])* [','] * vfpdef ['=' test] (',' vfpdef ['=' test])* [',']
*/ */
/* strip trailing comma node */ /* strip trailing comma node */
if (sym == COMMA) { if (sym == COMMA) {
@ -1258,9 +1297,9 @@ validate_varargslist(node *tree)
--nch; --nch;
} }
/* /*
* fpdef ['=' test] (',' fpdef ['=' test])* * vfpdef ['=' test] (',' vfpdef ['=' test])*
*/ */
res = validate_fpdef(CHILD(tree, 0)); res = validate_vfpdef(CHILD(tree, 0));
++i; ++i;
if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) { if (res && (i+2 <= nch) && TYPE(CHILD(tree, i)) == EQUAL) {
res = (validate_equal(CHILD(tree, i)) res = (validate_equal(CHILD(tree, i))
@ -1268,12 +1307,12 @@ validate_varargslist(node *tree)
i += 2; i += 2;
} }
/* /*
* ... (',' fpdef ['=' test])* * ... (',' vfpdef ['=' test])*
* i ---^^^ * i ---^^^
*/ */
while (res && (nch - i) >= 2) { while (res && (nch - i) >= 2) {
res = (validate_comma(CHILD(tree, i)) res = (validate_comma(CHILD(tree, i))
&& validate_fpdef(CHILD(tree, i+1))); && validate_vfpdef(CHILD(tree, i+1)));
i += 2; i += 2;
if (res && (nch - i) >= 2 && TYPE(CHILD(tree, i)) == EQUAL) { if (res && (nch - i) >= 2 && TYPE(CHILD(tree, i)) == EQUAL) {
res = (validate_equal(CHILD(tree, i)) res = (validate_equal(CHILD(tree, i))
@ -1405,24 +1444,32 @@ validate_gen_if(node *tree)
return res; return res;
} }
/* validate_fpdef() /* validate_vfpdef()
*
* Validate vfpdef or tfpdef.
*
* vname: NAME
* vfpdef: vname | '(' vfplist ')'
* vfplist: vfpdef (',' vfpdef)* [',']
*
* tname: NAME [':' test]
* tfpdef: tname | '(' tfplist ')'
* tfplist: tfpdef (',' tfpdef)* [',']
* *
* fpdef:
* NAME
* | '(' fplist ')'
*/ */
static int static int
validate_fpdef(node *tree) validate_vfpdef(node *tree)
{ {
int nch = NCH(tree); int nch = NCH(tree);
int res = validate_ntype(tree, fpdef); int typ = TYPE(tree);
int res = typ == vfpdef || typ == tfpdef;
if (res) { if (res) {
if (nch == 1) if (nch == 1)
res = validate_ntype(CHILD(tree, 0), NAME); res = validate_vname(CHILD(tree, 0));
else if (nch == 3) else if (nch == 3)
res = (validate_lparen(CHILD(tree, 0)) res = (validate_lparen(CHILD(tree, 0))
&& validate_fplist(CHILD(tree, 1)) && validate_vfplist(CHILD(tree, 1))
&& validate_rparen(CHILD(tree, 2))); && validate_rparen(CHILD(tree, 2)));
else else
res = validate_numnodes(tree, 1, "fpdef"); res = validate_numnodes(tree, 1, "fpdef");
@ -1432,10 +1479,10 @@ validate_fpdef(node *tree)
static int static int
validate_fplist(node *tree) validate_vfplist(node *tree)
{ {
return (validate_repeating_list(tree, fplist, return (validate_repeating_list(tree, vfplist,
validate_fpdef, "fplist")); validate_vfpdef, "vfplist"));
} }

71
Objects/funcobject.c
View File

@ -38,6 +38,7 @@ PyFunction_New(PyObject *code, PyObject *globals)
op->func_doc = doc; op->func_doc = doc;
op->func_dict = NULL; op->func_dict = NULL;
op->func_module = NULL; op->func_module = NULL;
op->func_annotations = NULL;
/* __module__: If module name is in globals, use it. /* __module__: If module name is in globals, use it.
Otherwise, use None. Otherwise, use None.
@ -187,6 +188,38 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure)
return 0; return 0;
} }
PyObject *
PyFunction_GetAnnotations(PyObject *op)
{
if (!PyFunction_Check(op)) {
PyErr_BadInternalCall();
return NULL;
}
return ((PyFunctionObject *) op) -> func_annotations;
}
int
PyFunction_SetAnnotations(PyObject *op, PyObject *annotations)
{
if (!PyFunction_Check(op)) {
PyErr_BadInternalCall();
return -1;
}
if (annotations == Py_None)
annotations = NULL;
else if (annotations && PyDict_Check(annotations)) {
Py_INCREF(annotations);
}
else {
PyErr_SetString(PyExc_SystemError,
"non-dict annotations");
return -1;
}
Py_XDECREF(((PyFunctionObject *)op) -> func_annotations);
((PyFunctionObject *) op) -> func_annotations = annotations;
return 0;
}
/* Methods */ /* Methods */
#define OFF(x) offsetof(PyFunctionObject, x) #define OFF(x) offsetof(PyFunctionObject, x)
@ -395,12 +428,48 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value)
return 0; return 0;
} }
static PyObject *
func_get_annotations(PyFunctionObject *op)
{
if (op->func_annotations == NULL) {
op->func_annotations = PyDict_New();
if (op->func_annotations == NULL)
return NULL;
}
Py_INCREF(op->func_annotations);
return op->func_annotations;
}
static int
func_set_annotations(PyFunctionObject *op, PyObject *value)
{
PyObject *tmp;
if (value == Py_None)
value = NULL;
/* Legal to del f.func_annotations.
* Can only set func_annotations to NULL (through C api)
* or a dict. */
if (value != NULL && !PyDict_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"func_annotations must be set to a dict object");
return -1;
}
tmp = op->func_annotations;
Py_XINCREF(value);
op->func_annotations = value;
Py_XDECREF(tmp);
return 0;
}
static PyGetSetDef func_getsetlist[] = { static PyGetSetDef func_getsetlist[] = {
{"func_code", (getter)func_get_code, (setter)func_set_code}, {"func_code", (getter)func_get_code, (setter)func_set_code},
{"func_defaults", (getter)func_get_defaults, {"func_defaults", (getter)func_get_defaults,
(setter)func_set_defaults}, (setter)func_set_defaults},
{"func_kwdefaults", (getter)func_get_kwdefaults, {"func_kwdefaults", (getter)func_get_kwdefaults,
(setter)func_set_kwdefaults}, (setter)func_set_kwdefaults},
{"func_annotations", (getter)func_get_annotations,
(setter)func_set_annotations},
{"func_dict", (getter)func_get_dict, (setter)func_set_dict}, {"func_dict", (getter)func_get_dict, (setter)func_set_dict},
{"__dict__", (getter)func_get_dict, (setter)func_set_dict}, {"__dict__", (getter)func_get_dict, (setter)func_set_dict},
{"func_name", (getter)func_get_name, (setter)func_set_name}, {"func_name", (getter)func_get_name, (setter)func_set_name},
@ -524,6 +593,7 @@ func_dealloc(PyFunctionObject *op)
Py_XDECREF(op->func_doc); Py_XDECREF(op->func_doc);
Py_XDECREF(op->func_dict); Py_XDECREF(op->func_dict);
Py_XDECREF(op->func_closure); Py_XDECREF(op->func_closure);
Py_XDECREF(op->func_annotations);
PyObject_GC_Del(op); PyObject_GC_Del(op);
} }
@ -546,6 +616,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg)
Py_VISIT(f->func_name); Py_VISIT(f->func_name);
Py_VISIT(f->func_dict); Py_VISIT(f->func_dict);
Py_VISIT(f->func_closure); Py_VISIT(f->func_closure);
Py_VISIT(f->func_annotations);
return 0; return 0;
} }

10
Parser/Python.asdl
View File

@ -10,7 +10,7 @@ module Python version "$Revision$"
| Suite(stmt* body) | Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args, stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorators) stmt* body, expr* decorators, expr? returns)
| ClassDef(identifier name, expr* bases, stmt* body) | ClassDef(identifier name, expr* bases, stmt* body)
| Return(expr? value) | Return(expr? value)
@ -100,8 +100,12 @@ module Python version "$Revision$"
excepthandler = (expr? type, expr? name, stmt* body, int lineno, excepthandler = (expr? type, expr? name, stmt* body, int lineno,
int col_offset) int col_offset)
arguments = (expr* args, identifier? vararg, expr* kwonlyargs, arguments = (arg* args, identifier? vararg, expr? varargannotation,
identifier? kwarg, expr* defaults, expr* kw_defaults) arg* kwonlyargs, identifier? kwarg,
expr? kwargannotation, expr* defaults,
expr* kw_defaults)
arg = SimpleArg(identifier arg, expr? annotation)
| NestedArgs(arg* args)
-- keyword arguments supplied to call -- keyword arguments supplied to call
keyword = (identifier arg, expr value) keyword = (identifier arg, expr value)

2
Parser/tokenizer.c
View File

@ -92,6 +92,7 @@ char *_PyParser_TokenNames[] = {
"DOUBLESLASH", "DOUBLESLASH",
"DOUBLESLASHEQUAL", "DOUBLESLASHEQUAL",
"AT", "AT",
"RARROW",
/* This table must match the #defines in token.h! */ /* This table must match the #defines in token.h! */
"OP", "OP",
"<ERRORTOKEN>", "<ERRORTOKEN>",
@ -998,6 +999,7 @@ PyToken_TwoChars(int c1, int c2)
case '-': case '-':
switch (c2) { switch (c2) {
case '=': return MINEQUAL; case '=': return MINEQUAL;
case '>': return RARROW;
} }
break; break;
case '*': case '*':

139
Python/Python-ast.c
View File

@ -34,6 +34,7 @@ static char *FunctionDef_fields[]={
"args", "args",
"body", "body",
"decorators", "decorators",
"returns",
}; };
static PyTypeObject *ClassDef_type; static PyTypeObject *ClassDef_type;
static char *ClassDef_fields[]={ static char *ClassDef_fields[]={
@ -333,11 +334,24 @@ static PyObject* ast2obj_arguments(void*);
static char *arguments_fields[]={ static char *arguments_fields[]={
"args", "args",
"vararg", "vararg",
"varargannotation",
"kwonlyargs", "kwonlyargs",
"kwarg", "kwarg",
"kwargannotation",
"defaults", "defaults",
"kw_defaults", "kw_defaults",
}; };
static PyTypeObject *arg_type;
static PyObject* ast2obj_arg(void*);
static PyTypeObject *SimpleArg_type;
static char *SimpleArg_fields[]={
"arg",
"annotation",
};
static PyTypeObject *NestedArgs_type;
static char *NestedArgs_fields[]={
"args",
};
static PyTypeObject *keyword_type; static PyTypeObject *keyword_type;
static PyObject* ast2obj_keyword(void*); static PyObject* ast2obj_keyword(void*);
static char *keyword_fields[]={ static char *keyword_fields[]={
@ -454,7 +468,7 @@ static int init_types(void)
if (!stmt_type) return 0; if (!stmt_type) return 0;
if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_type = make_type("FunctionDef", stmt_type,
FunctionDef_fields, 4); FunctionDef_fields, 5);
if (!FunctionDef_type) return 0; if (!FunctionDef_type) return 0;
ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 3); ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 3);
if (!ClassDef_type) return 0; if (!ClassDef_type) return 0;
@ -710,8 +724,16 @@ static int init_types(void)
excepthandler_type = make_type("excepthandler", AST_type, excepthandler_type = make_type("excepthandler", AST_type,
excepthandler_fields, 5); excepthandler_fields, 5);
if (!excepthandler_type) return 0; if (!excepthandler_type) return 0;
arguments_type = make_type("arguments", AST_type, arguments_fields, 6); arguments_type = make_type("arguments", AST_type, arguments_fields, 8);
if (!arguments_type) return 0; if (!arguments_type) return 0;
arg_type = make_type("arg", AST_type, NULL, 0);
if (!arg_type) return 0;
if (!add_attributes(arg_type, NULL, 0)) return 0;
SimpleArg_type = make_type("SimpleArg", arg_type, SimpleArg_fields, 2);
if (!SimpleArg_type) return 0;
NestedArgs_type = make_type("NestedArgs", arg_type, NestedArgs_fields,
1);
if (!NestedArgs_type) return 0;
keyword_type = make_type("keyword", AST_type, keyword_fields, 2); keyword_type = make_type("keyword", AST_type, keyword_fields, 2);
if (!keyword_type) return 0; if (!keyword_type) return 0;
alias_type = make_type("alias", AST_type, alias_fields, 2); alias_type = make_type("alias", AST_type, alias_fields, 2);
@ -783,7 +805,8 @@ Suite(asdl_seq * body, PyArena *arena)
stmt_ty stmt_ty
FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
decorators, int lineno, int col_offset, PyArena *arena) decorators, expr_ty returns, int lineno, int col_offset, PyArena
*arena)
{ {
stmt_ty p; stmt_ty p;
if (!name) { if (!name) {
@ -806,6 +829,7 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
p->v.FunctionDef.args = args; p->v.FunctionDef.args = args;
p->v.FunctionDef.body = body; p->v.FunctionDef.body = body;
p->v.FunctionDef.decorators = decorators; p->v.FunctionDef.decorators = decorators;
p->v.FunctionDef.returns = returns;
p->lineno = lineno; p->lineno = lineno;
p->col_offset = col_offset; p->col_offset = col_offset;
return p; return p;
@ -1830,8 +1854,9 @@ excepthandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int
} }
arguments_ty arguments_ty
arguments(asdl_seq * args, identifier vararg, asdl_seq * kwonlyargs, identifier arguments(asdl_seq * args, identifier vararg, expr_ty varargannotation,
kwarg, asdl_seq * defaults, asdl_seq * kw_defaults, PyArena *arena) asdl_seq * kwonlyargs, identifier kwarg, expr_ty kwargannotation,
asdl_seq * defaults, asdl_seq * kw_defaults, PyArena *arena)
{ {
arguments_ty p; arguments_ty p;
p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p)); p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p));
@ -1841,13 +1866,49 @@ arguments(asdl_seq * args, identifier vararg, asdl_seq * kwonlyargs, identifier
} }
p->args = args; p->args = args;
p->vararg = vararg; p->vararg = vararg;
p->varargannotation = varargannotation;
p->kwonlyargs = kwonlyargs; p->kwonlyargs = kwonlyargs;
p->kwarg = kwarg; p->kwarg = kwarg;
p->kwargannotation = kwargannotation;
p->defaults = defaults; p->defaults = defaults;
p->kw_defaults = kw_defaults; p->kw_defaults = kw_defaults;
return p; return p;
} }
arg_ty
SimpleArg(identifier arg, expr_ty annotation, PyArena *arena)
{
arg_ty p;
if (!arg) {
PyErr_SetString(PyExc_ValueError,
"field arg is required for SimpleArg");
return NULL;
}
p = (arg_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p) {
PyErr_NoMemory();
return NULL;
}
p->kind = SimpleArg_kind;
p->v.SimpleArg.arg = arg;
p->v.SimpleArg.annotation = annotation;
return p;
}
arg_ty
NestedArgs(asdl_seq * args, PyArena *arena)
{
arg_ty p;
p = (arg_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p) {
PyErr_NoMemory();
return NULL;
}
p->kind = NestedArgs_kind;
p->v.NestedArgs.args = args;
return p;
}
keyword_ty keyword_ty
keyword(identifier arg, expr_ty value, PyArena *arena) keyword(identifier arg, expr_ty value, PyArena *arena)
{ {
@ -1981,6 +2042,11 @@ ast2obj_stmt(void* _o)
if (PyObject_SetAttrString(result, "decorators", value) == -1) if (PyObject_SetAttrString(result, "decorators", value) == -1)
goto failed; goto failed;
Py_DECREF(value); Py_DECREF(value);
value = ast2obj_expr(o->v.FunctionDef.returns);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "returns", value) == -1)
goto failed;
Py_DECREF(value);
break; break;
case ClassDef_kind: case ClassDef_kind:
result = PyType_GenericNew(ClassDef_type, NULL, NULL); result = PyType_GenericNew(ClassDef_type, NULL, NULL);
@ -2901,7 +2967,7 @@ ast2obj_arguments(void* _o)
result = PyType_GenericNew(arguments_type, NULL, NULL); result = PyType_GenericNew(arguments_type, NULL, NULL);
if (!result) return NULL; if (!result) return NULL;
value = ast2obj_list(o->args, ast2obj_expr); value = ast2obj_list(o->args, ast2obj_arg);
if (!value) goto failed; if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1) if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed; goto failed;
@ -2911,7 +2977,12 @@ ast2obj_arguments(void* _o)
if (PyObject_SetAttrString(result, "vararg", value) == -1) if (PyObject_SetAttrString(result, "vararg", value) == -1)
goto failed; goto failed;
Py_DECREF(value); Py_DECREF(value);
value = ast2obj_list(o->kwonlyargs, ast2obj_expr); value = ast2obj_expr(o->varargannotation);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "varargannotation", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->kwonlyargs, ast2obj_arg);
if (!value) goto failed; if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1) if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1)
goto failed; goto failed;
@ -2921,6 +2992,11 @@ ast2obj_arguments(void* _o)
if (PyObject_SetAttrString(result, "kwarg", value) == -1) if (PyObject_SetAttrString(result, "kwarg", value) == -1)
goto failed; goto failed;
Py_DECREF(value); Py_DECREF(value);
value = ast2obj_expr(o->kwargannotation);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwargannotation", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->defaults, ast2obj_expr); value = ast2obj_list(o->defaults, ast2obj_expr);
if (!value) goto failed; if (!value) goto failed;
if (PyObject_SetAttrString(result, "defaults", value) == -1) if (PyObject_SetAttrString(result, "defaults", value) == -1)
@ -2938,6 +3014,48 @@ failed:
return NULL; return NULL;
} }
PyObject*
ast2obj_arg(void* _o)
{
arg_ty o = (arg_ty)_o;
PyObject *result = NULL, *value = NULL;
if (!o) {
Py_INCREF(Py_None);
return Py_None;
}
switch (o->kind) {
case SimpleArg_kind:
result = PyType_GenericNew(SimpleArg_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_identifier(o->v.SimpleArg.arg);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "arg", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_expr(o->v.SimpleArg.annotation);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "annotation", value) == -1)
goto failed;
Py_DECREF(value);
break;
case NestedArgs_kind:
result = PyType_GenericNew(NestedArgs_type, NULL, NULL);
if (!result) goto failed;
value = ast2obj_list(o->v.NestedArgs.args, ast2obj_arg);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed;
Py_DECREF(value);
break;
}
return result;
failed:
Py_XDECREF(value);
Py_XDECREF(result);
return NULL;
}
PyObject* PyObject*
ast2obj_keyword(void* _o) ast2obj_keyword(void* _o)
{ {
@ -3008,7 +3126,7 @@ init_ast(void)
if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return; if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return;
if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)
return; return;
if (PyModule_AddStringConstant(m, "__version__", "51773") < 0) if (PyModule_AddStringConstant(m, "__version__", "52491") < 0)
return; return;
if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return;
if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0)
@ -3146,6 +3264,11 @@ init_ast(void)
(PyObject*)excepthandler_type) < 0) return; (PyObject*)excepthandler_type) < 0) return;
if (PyDict_SetItemString(d, "arguments", (PyObject*)arguments_type) < if (PyDict_SetItemString(d, "arguments", (PyObject*)arguments_type) <
0) return; 0) return;
if (PyDict_SetItemString(d, "arg", (PyObject*)arg_type) < 0) return;
if (PyDict_SetItemString(d, "SimpleArg", (PyObject*)SimpleArg_type) <
0) return;
if (PyDict_SetItemString(d, "NestedArgs", (PyObject*)NestedArgs_type) <
0) return;
if (PyDict_SetItemString(d, "keyword", (PyObject*)keyword_type) < 0) if (PyDict_SetItemString(d, "keyword", (PyObject*)keyword_type) < 0)
return; return;
if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return; if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return;

303
Python/ast.c
View File

@ -388,14 +388,14 @@ set_context(expr_ty e, expr_context_ty ctx, const node *n)
expr_name = "list comprehension"; expr_name = "list comprehension";
break; break;
case Dict_kind: case Dict_kind:
case Set_kind: case Set_kind:
case Num_kind: case Num_kind:
case Str_kind: case Str_kind:
expr_name = "literal"; expr_name = "literal";
break; break;
case Ellipsis_kind: case Ellipsis_kind:
expr_name = "Ellipsis"; expr_name = "Ellipsis";
break; break;
case Compare_kind: case Compare_kind:
expr_name = "comparison"; expr_name = "comparison";
break; break;
@ -553,59 +553,74 @@ seq_for_testlist(struct compiling *c, const node *n)
return seq; return seq;
} }
static expr_ty static arg_ty
compiler_simple_arg(struct compiling *c, const node *n)
{
identifier name;
expr_ty annotation = NULL;
node *ch;
assert(TYPE(n) == tname || TYPE(n) == vname);
ch = CHILD(n, 0);
if (!strcmp(STR(ch), "None")) {
ast_error(ch, "assignment to None");
return NULL;
}
name = NEW_IDENTIFIER(ch);
if (!name)
return NULL;
if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {
annotation = ast_for_expr(c, CHILD(n, 2));
if (!annotation)
return NULL;
}
return SimpleArg(name, annotation, c->c_arena);
}
static arg_ty
compiler_complex_args(struct compiling *c, const node *n) compiler_complex_args(struct compiling *c, const node *n)
{ {
int i, len = (NCH(n) + 1) / 2; int i, len = (NCH(n) + 1) / 2;
expr_ty result; arg_ty arg;
asdl_seq *args = asdl_seq_new(len, c->c_arena); asdl_seq *args = asdl_seq_new(len, c->c_arena);
if (!args) if (!args)
return NULL; return NULL;
/* fpdef: NAME | '(' fplist ')' assert(TYPE(n) == tfplist || TYPE(n) == vfplist);
fplist: fpdef (',' fpdef)* [',']
*/
REQ(n, fplist);
for (i = 0; i < len; i++) { for (i = 0; i < len; i++) {
const node *fpdef_node = CHILD(n, 2*i); const node *child = CHILD(n, 2*i);
const node *child; /* def foo(((x), y)): -- x is not nested complex, special case. */
expr_ty arg; while (NCH(child) == 3 && NCH(CHILD(child, 1)) == 1)
set_name: child = CHILD(CHILD(child, 1), 0);
/* fpdef_node is either a NAME or an fplist */
child = CHILD(fpdef_node, 0); /* child either holds a tname or '(', a tfplist, ')' */
if (TYPE(child) == NAME) { switch (TYPE(CHILD(child, 0))) {
if (!strcmp(STR(child), "None")) { case tname:
ast_error(child, "assignment to None"); case vname:
return NULL; arg = compiler_simple_arg(c, CHILD(child, 0));
} break;
arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), case LPAR:
child->n_col_offset, c->c_arena); arg = compiler_complex_args(c, CHILD(child, 1));
} break;
else { default:
assert(TYPE(fpdef_node) == fpdef); PyErr_Format(PyExc_SystemError,
/* fpdef_node[0] is not a name, so it must be a '(', get CHILD[1] */ "unexpected node in args: %d @ %d",
child = CHILD(fpdef_node, 1); TYPE(CHILD(child, 0)), i);
assert(TYPE(child) == fplist); arg = NULL;
/* NCH == 1 means we have (x), we need to elide the extra parens */
if (NCH(child) == 1) {
fpdef_node = CHILD(child, 0);
assert(TYPE(fpdef_node) == fpdef);
goto set_name;
}
arg = compiler_complex_args(c, child);
} }
if (!arg)
return NULL;
asdl_seq_SET(args, i, arg); asdl_seq_SET(args, i, arg);
} }
result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena); return NestedArgs(args, c->c_arena);
if (!set_context(result, Store, n))
return NULL;
return result;
} }
/* returns -1 if failed to handle keyword only arguments /* returns -1 if failed to handle keyword only arguments
returns new position to keep processing if successful returns new position to keep processing if successful
(',' NAME ['=' test])* (',' tname ['=' test])*
^^^ ^^^
start pointing here start pointing here
*/ */
@ -614,7 +629,8 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
asdl_seq *kwonlyargs, asdl_seq *kwdefaults) asdl_seq *kwonlyargs, asdl_seq *kwdefaults)
{ {
node *ch; node *ch;
expr_ty name; expr_ty expression, annotation;
arg_ty arg;
int i = start; int i = start;
int j = 0; /* index for kwdefaults and kwonlyargs */ int j = 0; /* index for kwdefaults and kwonlyargs */
assert(kwonlyargs != NULL); assert(kwonlyargs != NULL);
@ -622,9 +638,10 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
while (i < NCH(n)) { while (i < NCH(n)) {
ch = CHILD(n, i); ch = CHILD(n, i);
switch (TYPE(ch)) { switch (TYPE(ch)) {
case NAME: case vname:
case tname:
if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
expr_ty expression = ast_for_expr(c, CHILD(n, i + 2)); expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression) { if (!expression) {
ast_error(ch, "assignment to None"); ast_error(ch, "assignment to None");
goto error; goto error;
@ -635,18 +652,28 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
else { /* setting NULL if no default value exists */ else { /* setting NULL if no default value exists */
asdl_seq_SET(kwdefaults, j, NULL); asdl_seq_SET(kwdefaults, j, NULL);
} }
if (NCH(ch) == 3) {
/* ch is NAME ':' test */
annotation = ast_for_expr(c, CHILD(ch, 2));
if (!annotation) {
ast_error(ch, "expected expression");
goto error;
}
}
else {
annotation = NULL;
}
ch = CHILD(ch, 0);
if (!strcmp(STR(ch), "None")) { if (!strcmp(STR(ch), "None")) {
ast_error(ch, "assignment to None"); ast_error(ch, "assignment to None");
goto error; goto error;
} }
name = Name(NEW_IDENTIFIER(ch), arg = SimpleArg(NEW_IDENTIFIER(ch), annotation, c->c_arena);
Param, LINENO(ch), ch->n_col_offset, if (!arg) {
c->c_arena);
if (!name) {
ast_error(ch, "expecting name"); ast_error(ch, "expecting name");
goto error; goto error;
} }
asdl_seq_SET(kwonlyargs, j++, name); asdl_seq_SET(kwonlyargs, j++, arg);
i += 2; /* the name and the comma */ i += 2; /* the name and the comma */
break; break;
case DOUBLESTAR: case DOUBLESTAR:
@ -666,29 +693,41 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
static arguments_ty static arguments_ty
ast_for_arguments(struct compiling *c, const node *n) ast_for_arguments(struct compiling *c, const node *n)
{ {
/* parameters: '(' [varargslist] ')' /* This function handles both typedargslist (function definition)
varargslist: (fpdef ['=' test] ',')* and varargslist (lambda definition).
('*' [NAME] (',' fpdef ['=' test])* [',' '**' NAME] | '**' NAME)
| fpdef ['=' test] (',' fpdef ['=' test])* [','] parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
('*' [tname] (',' tname ['=' test])* [',' '**' tname]
| '**' tname)
| tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
varargslist: ((vfpdef ['=' test] ',')*
('*' [vname] (',' vname ['=' test])* [',' '**' vname]
| '**' vname)
| vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
*/ */
int i, j, k, nposargs = 0, nkwonlyargs = 0; int i, j, k, nposargs = 0, nkwonlyargs = 0;
int nposdefaults = 0, found_default = 0; int nposdefaults = 0, found_default = 0;
asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults; asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
identifier vararg = NULL, kwarg = NULL; identifier vararg = NULL, kwarg = NULL;
arg_ty arg;
expr_ty varargannotation = NULL, kwargannotation = NULL;
node *ch; node *ch;
if (TYPE(n) == parameters) { if (TYPE(n) == parameters) {
if (NCH(n) == 2) /* () as argument list */ if (NCH(n) == 2) /* () as argument list */
return arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena); return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, c->c_arena);
n = CHILD(n, 1); n = CHILD(n, 1);
} }
REQ(n, varargslist); assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);
/* first count the number of positional args & defaults */ /* first count the number of positional args & defaults */
for (i = 0; i < NCH(n); i++) { for (i = 0; i < NCH(n); i++) {
ch = CHILD(n, i); ch = CHILD(n, i);
if (TYPE(ch) == STAR) { if (TYPE(ch) == STAR) {
if (TYPE(CHILD(n, i+1)) == NAME) { if (TYPE(CHILD(n, i+1)) == tname
|| TYPE(CHILD(n, i+1)) == vname) {
/* skip NAME of vararg */ /* skip NAME of vararg */
/* so that following can count only keyword only args */ /* so that following can count only keyword only args */
i += 2; i += 2;
@ -698,7 +737,7 @@ ast_for_arguments(struct compiling *c, const node *n)
} }
break; break;
} }
if (TYPE(ch) == fpdef) nposargs++; if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
if (TYPE(ch) == EQUAL) nposdefaults++; if (TYPE(ch) == EQUAL) nposdefaults++;
} }
/* count the number of keyword only args & /* count the number of keyword only args &
@ -706,35 +745,39 @@ ast_for_arguments(struct compiling *c, const node *n)
for ( ; i < NCH(n); ++i) { for ( ; i < NCH(n); ++i) {
ch = CHILD(n, i); ch = CHILD(n, i);
if (TYPE(ch) == DOUBLESTAR) break; if (TYPE(ch) == DOUBLESTAR) break;
if (TYPE(ch) == NAME) nkwonlyargs++; if (TYPE(ch) == tname || TYPE(ch) == vname) nkwonlyargs++;
} }
posargs = (nposargs ? asdl_seq_new(nposargs, c->c_arena) : NULL); posargs = (nposargs ? asdl_seq_new(nposargs, c->c_arena) : NULL);
if (!posargs && nposargs) if (!posargs && nposargs)
return NULL; /* Don't need to goto error; no objects allocated */ goto error;
kwonlyargs = (nkwonlyargs ? kwonlyargs = (nkwonlyargs ?
asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
if (!kwonlyargs && nkwonlyargs) if (!kwonlyargs && nkwonlyargs)
return NULL; /* Don't need to goto error; no objects allocated */ goto error;
posdefaults = (nposdefaults ? posdefaults = (nposdefaults ?
asdl_seq_new(nposdefaults, c->c_arena) : NULL); asdl_seq_new(nposdefaults, c->c_arena) : NULL);
if (!posdefaults && nposdefaults) if (!posdefaults && nposdefaults)
return NULL; /* Don't need to goto error; no objects allocated */ goto error;
/* The length of kwonlyargs and kwdefaults are same /* The length of kwonlyargs and kwdefaults are same
since we set NULL as default for keyword only argument w/o default since we set NULL as default for keyword only argument w/o default
- we have sequence data structure, but no dictionary */ - we have sequence data structure, but no dictionary */
kwdefaults = (nkwonlyargs ? kwdefaults = (nkwonlyargs ?
asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
if (!kwdefaults && nkwonlyargs) if (!kwdefaults && nkwonlyargs)
return NULL; /* Don't need to goto error; no objects allocated */ goto error;
if (nposargs + nkwonlyargs > 255) { if (nposargs + nkwonlyargs > 255) {
ast_error(n, "more than 255 arguments"); ast_error(n, "more than 255 arguments");
return NULL; return NULL;
} }
/* fpdef: NAME | '(' fplist ')' /* tname: NAME [':' test]
fplist: fpdef (',' fpdef)* [','] tfpdef: tname | '(' tfplist ')'
tfplist: tfpdef (',' tfpdef)* [',']
vname: NAME
vfpdef: NAME | '(' vfplist ')'
vfplist: vfpdef (',' vfpdef)* [',']
*/ */
i = 0; i = 0;
j = 0; /* index for defaults */ j = 0; /* index for defaults */
@ -742,8 +785,8 @@ ast_for_arguments(struct compiling *c, const node *n)
while (i < NCH(n)) { while (i < NCH(n)) {
ch = CHILD(n, i); ch = CHILD(n, i);
switch (TYPE(ch)) { switch (TYPE(ch)) {
case fpdef: case tfpdef:
handle_fpdef: case vfpdef:
/* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
anything other than EQUAL or a comma? */ anything other than EQUAL or a comma? */
/* XXX Should NCH(n) check be made a separate check? */ /* XXX Should NCH(n) check be made a separate check? */
@ -753,7 +796,6 @@ ast_for_arguments(struct compiling *c, const node *n)
goto error; goto error;
assert(posdefaults != NULL); assert(posdefaults != NULL);
asdl_seq_SET(posdefaults, j++, expression); asdl_seq_SET(posdefaults, j++, expression);
i += 2; i += 2;
found_default = 1; found_default = 1;
} }
@ -762,59 +804,47 @@ ast_for_arguments(struct compiling *c, const node *n)
"non-default argument follows default argument"); "non-default argument follows default argument");
goto error; goto error;
} }
if (NCH(ch) == 3) { /* def foo((x)): is not complex, special case. */
ch = CHILD(ch, 1); while (NCH(ch) == 3 && NCH(CHILD(ch, 1)) == 1)
/* def foo((x)): is not complex, special case. */ ch = CHILD(CHILD(ch, 1), 0);
if (NCH(ch) != 1) {
/* We have complex arguments, setup for unpacking. */ if (NCH(ch) != 1)
asdl_seq_SET(posargs, k++, arg = compiler_complex_args(c, CHILD(ch, 1));
compiler_complex_args(c, ch)); else
} else { arg = compiler_simple_arg(c, CHILD(ch, 0));
/* def foo((x)): setup for checking NAME below. */ if (!arg)
/* Loop because there can be many parens and tuple goto error;
unpacking mixed in. */ asdl_seq_SET(posargs, k++, arg);
ch = CHILD(ch, 0);
assert(TYPE(ch) == fpdef);
goto handle_fpdef;
}
}
if (TYPE(CHILD(ch, 0)) == NAME) {
expr_ty name;
if (!strcmp(STR(CHILD(ch, 0)), "None")) {
ast_error(CHILD(ch, 0), "assignment to None");
goto error;
}
name = Name(NEW_IDENTIFIER(CHILD(ch, 0)),
Param, LINENO(ch), ch->n_col_offset,
c->c_arena);
if (!name)
goto error;
asdl_seq_SET(posargs, k++, name);
}
i += 2; /* the name and the comma */ i += 2; /* the name and the comma */
break; break;
case STAR: case STAR:
if (i+1 >= NCH(n)) { if (i+1 >= NCH(n)) {
ast_error(CHILD(n, i), "no name for vararg"); ast_error(CHILD(n, i), "no name for vararg");
goto error; goto error;
} }
if (!strcmp(STR(CHILD(n, i+1)), "None")) { ch = CHILD(n, i+1); /* tname or COMMA */
ast_error(CHILD(n, i+1), "assignment to None"); if (TYPE(ch) == COMMA) {
goto error; int res = 0;
}
if (TYPE(CHILD(n, i+1)) == COMMA) {
int res = 0;
i += 2; /* now follows keyword only arguments */ i += 2; /* now follows keyword only arguments */
res = handle_keywordonly_args(c, n, i, res = handle_keywordonly_args(c, n, i,
kwonlyargs, kwdefaults); kwonlyargs, kwdefaults);
if (res == -1) goto error; if (res == -1) goto error;
i = res; /* res has new position to process */ i = res; /* res has new position to process */
} }
else if (!strcmp(STR(CHILD(ch, 0)), "None")) {
ast_error(CHILD(ch, 0), "assignment to None");
goto error;
}
else { else {
vararg = NEW_IDENTIFIER(CHILD(n, i+1)); vararg = NEW_IDENTIFIER(CHILD(ch, 0));
if (NCH(ch) > 1) {
/* there is an annotation on the vararg */
varargannotation = ast_for_expr(c, CHILD(ch, 2));
}
i += 3; i += 3;
if (i < NCH(n) && TYPE(CHILD(n, i)) == NAME) { if (i < NCH(n) && (TYPE(CHILD(n, i)) == tname
|| TYPE(CHILD(n, i)) == vname)) {
int res = 0; int res = 0;
res = handle_keywordonly_args(c, n, i, res = handle_keywordonly_args(c, n, i,
kwonlyargs, kwdefaults); kwonlyargs, kwdefaults);
@ -824,11 +854,17 @@ ast_for_arguments(struct compiling *c, const node *n)
} }
break; break;
case DOUBLESTAR: case DOUBLESTAR:
if (!strcmp(STR(CHILD(n, i+1)), "None")) { ch = CHILD(n, i+1); /* tname */
ast_error(CHILD(n, i+1), "assignment to None"); assert(TYPE(ch) == tname || TYPE(ch) == vname);
if (!strcmp(STR(CHILD(ch, 0)), "None")) {
ast_error(CHILD(ch, 0), "assignment to None");
goto error; goto error;
} }
kwarg = NEW_IDENTIFIER(CHILD(n, i+1)); kwarg = NEW_IDENTIFIER(CHILD(ch, 0));
if (NCH(ch) > 1) {
/* there is an annotation on the kwarg */
kwargannotation = ast_for_expr(c, CHILD(ch, 2));
}
i += 3; i += 3;
break; break;
default: default:
@ -838,8 +874,8 @@ ast_for_arguments(struct compiling *c, const node *n)
goto error; goto error;
} }
} }
return arguments(posargs, vararg, kwonlyargs, kwarg, return arguments(posargs, vararg, varargannotation, kwonlyargs, kwarg,
posdefaults, kwdefaults, c->c_arena); kwargannotation, posdefaults, kwdefaults, c->c_arena);
error: error:
Py_XDECREF(vararg); Py_XDECREF(vararg);
Py_XDECREF(kwarg); Py_XDECREF(kwarg);
@ -938,11 +974,12 @@ ast_for_decorators(struct compiling *c, const node *n)
static stmt_ty static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n) ast_for_funcdef(struct compiling *c, const node *n)
{ {
/* funcdef: 'def' [decorators] NAME parameters ':' suite */ /* funcdef: 'def' [decorators] NAME parameters ['->' test] ':' suite */
identifier name; identifier name;
arguments_ty args; arguments_ty args;
asdl_seq *body; asdl_seq *body;
asdl_seq *decorator_seq = NULL; asdl_seq *decorator_seq = NULL;
expr_ty returns = NULL;
int name_i; int name_i;
REQ(n, funcdef); REQ(n, funcdef);
@ -967,11 +1004,17 @@ ast_for_funcdef(struct compiling *c, const node *n)
args = ast_for_arguments(c, CHILD(n, name_i + 1)); args = ast_for_arguments(c, CHILD(n, name_i + 1));
if (!args) if (!args)
return NULL; return NULL;
if (TYPE(CHILD(n, name_i+2)) == RARROW) {
returns = ast_for_expr(c, CHILD(n, name_i + 3));
if (!returns)
return NULL;
name_i += 2;
}
body = ast_for_suite(c, CHILD(n, name_i + 3)); body = ast_for_suite(c, CHILD(n, name_i + 3));
if (!body) if (!body)
return NULL; return NULL;
return FunctionDef(name, args, body, decorator_seq, LINENO(n), return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n),
n->n_col_offset, c->c_arena); n->n_col_offset, c->c_arena);
} }
@ -983,7 +1026,8 @@ ast_for_lambdef(struct compiling *c, const node *n)
expr_ty expression; expr_ty expression;
if (NCH(n) == 3) { if (NCH(n) == 3) {
args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena); args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, c->c_arena);
if (!args) if (!args)
return NULL; return NULL;
expression = ast_for_expr(c, CHILD(n, 2)); expression = ast_for_expr(c, CHILD(n, 2));
@ -1361,9 +1405,8 @@ ast_for_atom(struct compiling *c, const node *n)
PyArena_AddPyObject(c->c_arena, pynum); PyArena_AddPyObject(c->c_arena, pynum);
return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
} }
case DOT: case DOT: /* Ellipsis */
/* Ellipsis */ return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
case LPAR: /* some parenthesized expressions */ case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1); ch = CHILD(n, 1);
@ -1394,13 +1437,13 @@ ast_for_atom(struct compiling *c, const node *n)
else else
return ast_for_listcomp(c, ch); return ast_for_listcomp(c, ch);
case LBRACE: { case LBRACE: {
/* dictsetmaker: test ':' test (',' test ':' test)* [','] | /* dictsetmaker: test ':' test (',' test ':' test)* [','] |
* test (',' test)* [','] */ * test (',' test)* [','] */
int i, size; int i, size;
asdl_seq *keys, *values; asdl_seq *keys, *values;
ch = CHILD(n, 1); ch = CHILD(n, 1);
if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) { if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) {
/* it's a set */ /* it's a set */
size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */ size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
keys = asdl_seq_new(size, c->c_arena); keys = asdl_seq_new(size, c->c_arena);
@ -3046,10 +3089,10 @@ ast_for_stmt(struct compiling *c, const node *n)
n = CHILD(n, 0); n = CHILD(n, 0);
} }
if (TYPE(n) == small_stmt) { if (TYPE(n) == small_stmt) {
REQ(n, small_stmt); REQ(n, small_stmt);
n = CHILD(n, 0); n = CHILD(n, 0);
/* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt /* small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt
| flow_stmt | import_stmt | global_stmt | assert_stmt | flow_stmt | import_stmt | global_stmt | assert_stmt
*/ */
switch (TYPE(n)) { switch (TYPE(n)) {
case expr_stmt: case expr_stmt:

28
Python/ceval.c
View File

@ -2293,10 +2293,37 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
{ {
int posdefaults = oparg & 0xff; int posdefaults = oparg & 0xff;
int kwdefaults = (oparg>>8) & 0xff; int kwdefaults = (oparg>>8) & 0xff;
int num_annotations = (oparg >> 16) & 0x7fff;
v = POP(); /* code object */ v = POP(); /* code object */
x = PyFunction_New(v, f->f_globals); x = PyFunction_New(v, f->f_globals);
Py_DECREF(v); Py_DECREF(v);
if (x != NULL && num_annotations > 0) {
Py_ssize_t name_ix;
u = POP(); /* names of args with annotations */
v = PyDict_New();
if (v == NULL) {
Py_DECREF(x);
x = NULL;
break;
}
name_ix = PyTuple_Size(u);
assert(num_annotations == name_ix+1);
while (name_ix > 0) {
--name_ix;
t = PyTuple_GET_ITEM(u, name_ix);
w = POP();
/* XXX(nnorwitz): check for errors */
PyDict_SetItem(v, t, w);
Py_DECREF(w);
}
err = PyFunction_SetAnnotations(x, v);
Py_DECREF(v);
Py_DECREF(u);
}
/* XXX Maybe this should be a separate opcode? */ /* XXX Maybe this should be a separate opcode? */
if (x != NULL && posdefaults > 0) { if (x != NULL && posdefaults > 0) {
v = PyTuple_New(posdefaults); v = PyTuple_New(posdefaults);
@ -2322,6 +2349,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
while (--kwdefaults >= 0) { while (--kwdefaults >= 0) {
w = POP(); /* default value */ w = POP(); /* default value */
u = POP(); /* kw only arg name */ u = POP(); /* kw only arg name */
/* XXX(nnorwitz): check for errors */
PyDict_SetItem(v, u, w); PyDict_SetItem(v, u, w);
} }
err = PyFunction_SetKwDefaults(x, v); err = PyFunction_SetKwDefaults(x, v);

141
Python/compile.c
View File

@ -832,7 +832,7 @@ opcode_stack_effect(int opcode, int oparg)
case RAISE_VARARGS: case RAISE_VARARGS:
return -oparg; return -oparg;
#define NARGS(o) (((o) % 256) + 2*((o) / 256)) #define NARGS(o) (((o) % 256) + 2*(((o) / 256) % 256))
case CALL_FUNCTION: case CALL_FUNCTION:
return -NARGS(oparg); return -NARGS(oparg);
case CALL_FUNCTION_VAR: case CALL_FUNCTION_VAR:
@ -841,7 +841,7 @@ opcode_stack_effect(int opcode, int oparg)
case CALL_FUNCTION_VAR_KW: case CALL_FUNCTION_VAR_KW:
return -NARGS(oparg)-2; return -NARGS(oparg)-2;
case MAKE_FUNCTION: case MAKE_FUNCTION:
return -NARGS(oparg); return -NARGS(oparg) - ((oparg >> 16) & 0xffff);
#undef NARGS #undef NARGS
case BUILD_SLICE: case BUILD_SLICE:
if (oparg == 3) if (oparg == 3)
@ -1266,15 +1266,38 @@ compiler_decorators(struct compiler *c, asdl_seq* decos)
return 1; return 1;
} }
static int
compiler_unpack_nested(struct compiler *c, asdl_seq *args) {
int i, len;
len = asdl_seq_LEN(args);
ADDOP_I(c, UNPACK_SEQUENCE, len);
for (i = 0; i < len; i++) {
arg_ty elt = (arg_ty)asdl_seq_GET(args, i);
switch (elt->kind) {
case SimpleArg_kind:
if (!compiler_nameop(c, elt->v.SimpleArg.arg, Store))
return 0;
break;
case NestedArgs_kind:
if (!compiler_unpack_nested(c, elt->v.NestedArgs.args))
return 0;
break;
default:
return 0;
}
}
return 1;
}
static int static int
compiler_arguments(struct compiler *c, arguments_ty args) compiler_arguments(struct compiler *c, arguments_ty args)
{ {
int i; int i;
int n = asdl_seq_LEN(args->args); int n = asdl_seq_LEN(args->args);
/* Correctly handle nested argument lists */
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
expr_ty arg = (expr_ty)asdl_seq_GET(args->args, i); arg_ty arg = (arg_ty)asdl_seq_GET(args->args, i);
if (arg->kind == Tuple_kind) { if (arg->kind == NestedArgs_kind) {
PyObject *id = PyString_FromFormat(".%d", i); PyObject *id = PyString_FromFormat(".%d", i);
if (id == NULL) { if (id == NULL) {
return 0; return 0;
@ -1284,7 +1307,8 @@ compiler_arguments(struct compiler *c, arguments_ty args)
return 0; return 0;
} }
Py_DECREF(id); Py_DECREF(id);
VISIT(c, expr, arg); if (!compiler_unpack_nested(c, arg->v.NestedArgs.args))
return 0;
} }
} }
return 1; return 1;
@ -1296,10 +1320,10 @@ compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs,
{ {
int i, default_count = 0; int i, default_count = 0;
for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
expr_ty arg = asdl_seq_GET(kwonlyargs, i); arg_ty arg = asdl_seq_GET(kwonlyargs, i);
expr_ty default_ = asdl_seq_GET(kw_defaults, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i);
if (default_) { if (default_) {
ADDOP_O(c, LOAD_CONST, arg->v.Name.id, consts); ADDOP_O(c, LOAD_CONST, arg->v.SimpleArg.arg, consts);
if (!compiler_visit_expr(c, default_)) { if (!compiler_visit_expr(c, default_)) {
return -1; return -1;
} }
@ -1309,15 +1333,113 @@ compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs,
return default_count; return default_count;
} }
static int
compiler_visit_argannotation(struct compiler *c, identifier id,
expr_ty annotation, PyObject *names)
{
if (annotation) {
VISIT(c, expr, annotation);
if (PyList_Append(names, id))
return -1;
}
return 0;
}
static int
compiler_visit_argannotations(struct compiler *c, asdl_seq* args,
PyObject *names)
{
int i, error;
for (i = 0; i < asdl_seq_LEN(args); i++) {
arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
if (arg->kind == NestedArgs_kind)
error = compiler_visit_argannotations(
c,
arg->v.NestedArgs.args,
names);
else
error = compiler_visit_argannotation(
c,
arg->v.SimpleArg.arg,
arg->v.SimpleArg.annotation,
names);
if (error)
return error;
}
return 0;
}
static int
compiler_visit_annotations(struct compiler *c, arguments_ty args,
expr_ty returns)
{
/* push arg annotations and a list of the argument names. return the #
of items pushed. this is out-of-order wrt the source code. */
static identifier return_str;
PyObject *names;
int len;
names = PyList_New(0);
if (!names)
return -1;
if (compiler_visit_argannotations(c, args->args, names))
goto error;
if (args->varargannotation &&
compiler_visit_argannotation(c, args->vararg,
args->varargannotation, names))
goto error;
if (compiler_visit_argannotations(c, args->kwonlyargs, names))
goto error;
if (args->kwargannotation &&
compiler_visit_argannotation(c, args->kwarg,
args->kwargannotation, names))
goto error;
if (!return_str) {
return_str = PyString_InternFromString("return");
if (!return_str)
goto error;
}
if (compiler_visit_argannotation(c, return_str, returns, names)) {
goto error;
}
len = PyList_GET_SIZE(names);
if (len) {
/* convert names to a tuple and place on stack */
PyObject *elt;
int i;
PyObject *s = PyTuple_New(len);
if (!s)
goto error;
for (i = 0; i < len; i++) {
elt = PyList_GET_ITEM(names, i);
Py_INCREF(elt);
PyTuple_SET_ITEM(s, i, elt);
}
ADDOP_O(c, LOAD_CONST, s, consts);
Py_DECREF(s);
len++; /* include the just-pushed tuple */
}
Py_DECREF(names);
return len;
error:
Py_DECREF(names);
return -1;
}
static int static int
compiler_function(struct compiler *c, stmt_ty s) compiler_function(struct compiler *c, stmt_ty s)
{ {
PyCodeObject *co; PyCodeObject *co;
PyObject *first_const = Py_None; PyObject *first_const = Py_None;
arguments_ty args = s->v.FunctionDef.args; arguments_ty args = s->v.FunctionDef.args;
expr_ty returns = s->v.FunctionDef.returns;
asdl_seq* decos = s->v.FunctionDef.decorators; asdl_seq* decos = s->v.FunctionDef.decorators;
stmt_ty st; stmt_ty st;
int i, n, docstring, kw_default_count = 0, arglength; int i, n, docstring, kw_default_count = 0, arglength;
int num_annotations;
assert(s->kind == FunctionDef_kind); assert(s->kind == FunctionDef_kind);
@ -1332,6 +1454,7 @@ compiler_function(struct compiler *c, stmt_ty s)
} }
if (args->defaults) if (args->defaults)
VISIT_SEQ(c, expr, args->defaults); VISIT_SEQ(c, expr, args->defaults);
num_annotations = compiler_visit_annotations(c, args, returns);
if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s, if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s,
s->lineno)) s->lineno))
@ -1364,9 +1487,11 @@ compiler_function(struct compiler *c, stmt_ty s)
arglength = asdl_seq_LEN(args->defaults); arglength = asdl_seq_LEN(args->defaults);
arglength |= kw_default_count << 8; arglength |= kw_default_count << 8;
arglength |= num_annotations << 16;
compiler_make_closure(c, co, arglength); compiler_make_closure(c, co, arglength);
Py_DECREF(co); Py_DECREF(co);
/* decorators */
for (i = 0; i < asdl_seq_LEN(decos); i++) { for (i = 0; i < asdl_seq_LEN(decos); i++) {
ADDOP_I(c, CALL_FUNCTION, 1); ADDOP_I(c, CALL_FUNCTION, 1);
} }

2969
Python/graminit.c
View File

File diff suppressed because it is too large Load Diff

73
Python/symtable.c
View File

@ -172,9 +172,12 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty);
static int symtable_visit_slice(struct symtable *st, slice_ty); static int symtable_visit_slice(struct symtable *st, slice_ty);
static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top); static int symtable_visit_params(struct symtable *st, asdl_seq *args, int top,
static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args); int annotations);
static int symtable_visit_params_nested(struct symtable *st, asdl_seq *args,
int annotations);
static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_implicit_arg(struct symtable *st, int pos);
static int symtable_visit_annotations(struct symtable *st, stmt_ty s);
static identifier top = NULL, lambda = NULL, genexpr = NULL; static identifier top = NULL, lambda = NULL, genexpr = NULL;
@ -935,6 +938,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
if (s->v.FunctionDef.args->kw_defaults) if (s->v.FunctionDef.args->kw_defaults)
VISIT_KWONLYDEFAULTS(st, VISIT_KWONLYDEFAULTS(st,
s->v.FunctionDef.args->kw_defaults); s->v.FunctionDef.args->kw_defaults);
if (!symtable_visit_annotations(st, s))
return 0;
if (s->v.FunctionDef.decorators) if (s->v.FunctionDef.decorators)
VISIT_SEQ(st, expr, s->v.FunctionDef.decorators); VISIT_SEQ(st, expr, s->v.FunctionDef.decorators);
if (!symtable_enter_block(st, s->v.FunctionDef.name, if (!symtable_enter_block(st, s->v.FunctionDef.name,
@ -1219,22 +1224,29 @@ symtable_implicit_arg(struct symtable *st, int pos)
} }
static int static int
symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel) symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel,
int annotations)
{ {
int i; int i;
if (!args)
return -1;
/* go through all the toplevel arguments first */ /* go through all the toplevel arguments first */
for (i = 0; i < asdl_seq_LEN(args); i++) { for (i = 0; i < asdl_seq_LEN(args); i++) {
expr_ty arg = (expr_ty)asdl_seq_GET(args, i); arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
if (arg->kind == Name_kind) { if (arg->kind == SimpleArg_kind) {
assert(arg->v.Name.ctx == Param || if (!annotations) {
(arg->v.Name.ctx == Store && !toplevel)); if (!symtable_add_def(st,
if (!symtable_add_def(st, arg->v.Name.id, DEF_PARAM)) arg->v.SimpleArg.arg,
return 0; DEF_PARAM))
return 0;
}
else if (arg->v.SimpleArg.annotation)
VISIT(st, expr, arg->v.SimpleArg.annotation);
} }
else if (arg->kind == Tuple_kind) { else if (arg->kind == NestedArgs_kind) {
assert(arg->v.Tuple.ctx == Store); if (toplevel && !annotations) {
if (toplevel) {
if (!symtable_implicit_arg(st, i)) if (!symtable_implicit_arg(st, i))
return 0; return 0;
} }
@ -1249,7 +1261,7 @@ symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel)
} }
if (!toplevel) { if (!toplevel) {
if (!symtable_visit_params_nested(st, args)) if (!symtable_visit_params_nested(st, args, annotations))
return 0; return 0;
} }
@ -1257,16 +1269,37 @@ symtable_visit_params(struct symtable *st, asdl_seq *args, int toplevel)
} }
static int static int
symtable_visit_params_nested(struct symtable *st, asdl_seq *args) symtable_visit_params_nested(struct symtable *st, asdl_seq *args,
int annotations)
{ {
int i; int i;
for (i = 0; i < asdl_seq_LEN(args); i++) { for (i = 0; i < asdl_seq_LEN(args); i++) {
expr_ty arg = (expr_ty)asdl_seq_GET(args, i); arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
if (arg->kind == Tuple_kind && if (arg->kind == NestedArgs_kind &&
!symtable_visit_params(st, arg->v.Tuple.elts, 0)) !symtable_visit_params(st, arg->v.NestedArgs.args, 0,
annotations))
return 0; return 0;
} }
return 1;
}
static int
symtable_visit_annotations(struct symtable *st, stmt_ty s)
{
arguments_ty a = s->v.FunctionDef.args;
if (a->args && !symtable_visit_params(st, a->args, 1, 1))
return 0;
if (a->varargannotation)
VISIT(st, expr, a->varargannotation);
if (a->kwargannotation)
VISIT(st, expr, a->kwargannotation);
if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs, 1, 1))
return 0;
if (s->v.FunctionDef.returns)
VISIT(st, expr, s->v.FunctionDef.returns);
return 1; return 1;
} }
@ -1276,9 +1309,9 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a)
/* skip default arguments inside function block /* skip default arguments inside function block
XXX should ast be different? XXX should ast be different?
*/ */
if (a->args && !symtable_visit_params(st, a->args, 1)) if (a->args && !symtable_visit_params(st, a->args, 1, 0))
return 0; return 0;
if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs, 1)) if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs, 1, 0))
return 0; return 0;
if (a->vararg) { if (a->vararg) {
if (!symtable_add_def(st, a->vararg, DEF_PARAM)) if (!symtable_add_def(st, a->vararg, DEF_PARAM))
@ -1290,7 +1323,7 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a)
return 0; return 0;
st->st_cur->ste_varkeywords = 1; st->st_cur->ste_varkeywords = 1;
} }
if (a->args && !symtable_visit_params_nested(st, a->args)) if (a->args && !symtable_visit_params_nested(st, a->args, 0))
return 0; return 0;
return 1; return 1;
} }

10
Tools/compiler/ast.txt
View File

@ -12,8 +12,11 @@
Module: doc*, node Module: doc*, node
Stmt: nodes! Stmt: nodes!
Decorators: nodes! Decorators: nodes!
Function: decorators&, name*, argnames*, defaults!, kwonlyargs*, flags*, doc*, code Function: decorators&, name*, arguments!, defaults!, kwonlyargs!, returns&, flags*, doc*, code
Lambda: argnames*, defaults!, kwonlyargs*, flags*, code Lambda: arguments!, defaults!, kwonlyargs!, flags*, code
SimpleArg: name*, annotation&
NestedArgs: args!
Kwarg: arg, expr
Class: name*, bases!, doc*, code Class: name*, bases!, doc*, code
Pass: Pass:
Break: Break:
@ -93,9 +96,10 @@ init(Lambda):
self.varargs = 1 self.varargs = 1
if flags & CO_VARKEYWORDS: if flags & CO_VARKEYWORDS:
self.kwargs = 1 self.kwargs = 1
self.returns = None
init(GenExpr): init(GenExpr):
self.argnames = ['.0'] self.arguments = [SimpleArg('.0', None)]
self.varargs = self.kwargs = None self.varargs = self.kwargs = None
self.kwonlyargs = () self.kwonlyargs = ()

5
Tools/compiler/astgen.py
View File

@ -266,7 +266,10 @@ class Node:
pass # implemented by subclasses pass # implemented by subclasses
class EmptyNode(Node): class EmptyNode(Node):
pass def getChildNodes(self):
return ()
def getChildren(self):
return ()
class Expression(Node): class Expression(Node):
# Expression is an artificial node class to support "eval" # Expression is an artificial node class to support "eval"