From f968e8545d5c7fddca22579aa5914c57620b2cd1 Mon Sep 17 00:00:00 2001 From: Jeremy Hylton Date: Fri, 4 Feb 2000 00:25:23 +0000 Subject: [PATCH] three files from the p2c cvs tree. the message here indicates the revision number the p2c cvs tree. COPYRIGHT: 1.1 ast.py: 1.3 transformer.py: 1.11 --- Lib/compiler/ast.py | 689 ++++++++++++++ Lib/compiler/transformer.py | 1204 ++++++++++++++++++++++++ Tools/compiler/compiler/ast.py | 689 ++++++++++++++ Tools/compiler/compiler/transformer.py | 1204 ++++++++++++++++++++++++ 4 files changed, 3786 insertions(+) create mode 100644 Lib/compiler/ast.py create mode 100644 Lib/compiler/transformer.py create mode 100644 Tools/compiler/compiler/ast.py create mode 100644 Tools/compiler/compiler/transformer.py diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py new file mode 100644 index 00000000000..9c214d453d1 --- /dev/null +++ b/Lib/compiler/ast.py @@ -0,0 +1,689 @@ +import types + +nodes = {} + +def flatten(list): + l = [] + for elt in list: + if type(elt) is types.TupleType: + for elt2 in flatten(elt): + l.append(elt2) + elif type(elt) is types.ListType: + for elt2 in flatten(elt): + l.append(elt2) + else: + l.append(elt) + return l + +def asList(nodes): + l = [] + for item in nodes: + if hasattr(item, "asList"): + l.append(item.asList()) + else: + if type(item) is types.TupleType: + l.append(tuple(asList(item))) + elif type(item) is types.ListType: + l.append(asList(item)) + else: + l.append(item) + return l + +class Node: + def __init__(self, *args): + self._children = args + self.lineno = None + def __getitem__(self, index): + return self._children[index] + def __repr__(self): + return "" % self._children[0] + def __len__(self): + return len(self._children) + def __getslice__(self, low, high): + return self._children[low:high] + def getChildren(self): + return tuple(flatten(self._children[1:])) + def getType(self): + return self._children[0] + def asList(self): + return tuple(asList(self._children)) + +class EmptyNode(Node): + def __init__(self): + self.lineno = None + +class Module(Node): + nodes['module'] = 'Module' + + def __init__(self, doc, node): + self.doc = doc + self.node = node + self._children = ('module', doc, node) + + def __repr__(self): + return "Module(%s,%s)" % self._children[1:] + +class Stmt(Node): + nodes['stmt'] = 'Stmt' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('stmt', nodes) + + def __repr__(self): + return "Stmt(%s)" % self._children[1:] + +class Function(Node): + nodes['function'] = 'Function' + + def __init__(self, name, argnames, defaults, flags, doc, code): + self.name = name + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.doc = doc + self.code = code + self._children = ('function', + name, argnames, defaults, flags, doc, code) + + def __repr__(self): + return "Function(%s,%s,%s,%s,%s,%s)" % self._children[1:] + +class Lambda(Node): + nodes['lambda'] = 'Lambda' + + def __init__(self, argnames, defaults, flags, code): + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.code = code + self._children = ('lambda', argnames, defaults, flags, code) + + def __repr__(self): + return "Lambda(%s,%s,%s,%s)" % self._children[1:] + +class Classdef(Node): + nodes['classdef'] = 'Classdef' + + def __init__(self, name, bases, doc, code): + self.name = name + self.bases = bases + self.doc = doc + self.code = code + self._children = ('classdef', name, bases, doc, code) + + def __repr__(self): + return "Classdef(%s,%s,%s,%s)" % self._children[1:] + +class Pass(EmptyNode): + nodes['pass'] = 'Pass' + _children = ('pass',) + def __repr__(self): + return "Pass()" + +class Break(EmptyNode): + nodes['break'] = 'Break' + _children = ('break',) + def __repr__(self): + return "Break()" + +class Continue(EmptyNode): + nodes['continue'] = 'Continue' + _children = ('continue',) + def __repr__(self): + return "Continue()" + +class For(Node): + nodes['for'] = 'For' + + def __init__(self, assign, list, body, else_): + self.assign = assign + self.list = list + self.body = body + self.else_ = else_ + self._children = ('for', assign, list, body, else_) + + def __repr__(self): + return "For(%s,%s,%s,%s)" % self._children[1:] + +class While(Node): + nodes['while'] = 'While' + + def __init__(self, test, body, else_): + self.test = test + self.body = body + self.else_ = else_ + self._children = ('while', test, body, else_) + + def __repr__(self): + return "While(%s,%s,%s)" % self._children[1:] + +class If(Node): + """if: [ (testNode, suiteNode), ...], elseNode""" + nodes['if'] = 'If' + + def __init__(self, tests, else_): + self.tests = tests + self.else_ = else_ + self._children = ('if', tests, else_) + + def __repr__(self): + return "If(%s,%s)" % self._children[1:] + +class Exec(Node): + nodes['exec'] = 'Exec' + + def __init__(self, expr1, expr2, expr3): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self._children = ('exec', expr1, expr2, expr3) + + def __repr__(self): + return "Exec(%s,%s,%s)" % self._children[1:] + +class From(Node): + nodes['from'] = 'From' + + def __init__(self, modname, names): + self.modname = modname + self.names = names + self._children = ('from', modname, names) + + def __repr__(self): + return "From(%s,%s)" % self._children[1:] + +class Import(Node): + nodes['import'] = 'Import' + + def __init__(self, names): + self.names = names + self._children = ('import', names) + + def __repr__(self): + return "Import(%s)" % self._children[1:] + +class Raise(Node): + nodes['raise'] = 'Raise' + + def __init__(self, expr1, expr2, expr3): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self._children = ('raise', expr1, expr2, expr3) + + def __repr__(self): + return "Raise(%s,%s,%s)" % self._children[1:] + +class Tryfinally(Node): + nodes['tryfinally'] = 'Tryfinally' + + def __init__(self, try_, fin): + self.try_ = try_ + self.fin = fin + self._children = ('tryfinally', try_, fin) + + def __repr__(self): + return "Tryfinally(%s,%s)" % self._children[1:] + +class Tryexcept(Node): + nodes['tryexcept'] = 'Tryexcept' + + def __init__(self, try_, excs, else_): + self.try_ = try_ + self.excs = excs + self.else_ = else_ + self._children = ('tryexcept', try_, excs, else_) + + def __repr__(self): + return "Tryexcept(%s,%s,%s)" % self._children[1:] + +class Return(Node): + nodes['return'] = 'Return' + + def __init__(self, value): + self.value = value + self._children = ('return', value) + + def __repr__(self): + return "Return(%s)" % self._children[1:] + +class Const(Node): + nodes['const'] = 'Const' + + def __init__(self, value): + self.value = value + self._children = ('const', value) + + def __repr__(self): + return "Const(%s)" % self._children[1:] + +class Print(Node): + nodes['print'] = 'Print' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('print', nodes) + + def __repr__(self): + return "Print(%s)" % self._children[1:] + +class Printnl(Node): + nodes['printnl'] = 'Printnl' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('printnl', nodes) + + def __repr__(self): + return "Printnl(%s)" % self._children[1:] + +class Discard(Node): + nodes['discard'] = 'Discard' + + def __init__(self, expr): + self.expr = expr + self._children = ('discard', expr) + + def __repr__(self): + return "Discard(%s)" % self._children[1:] + +class Assign(Node): + nodes['assign'] = 'Assign' + + def __init__(self, nodes, expr): + self.nodes = nodes + self.expr = expr + self._children = ('assign', nodes, expr) + + def __repr__(self): + return "Assign(%s,%s)" % self._children[1:] + +class AssTuple(Node): + nodes['ass_tuple'] = 'AssTuple' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('ass_tuple', nodes) + + def __repr__(self): + return "AssTuple(%s)" % self._children[1:] + +class AssList(Node): + nodes['ass_list'] = 'AssList' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('ass_list', nodes) + + def __repr__(self): + return "AssList(%s)" % self._children[1:] + +class AssName(Node): + nodes['ass_name'] = 'AssName' + + def __init__(self, name, flags): + self.name = name + self.flags = flags + self._children = ('ass_name', name, flags) + + def __repr__(self): + return "AssName(%s,%s)" % self._children[1:] + +class AssAttr(Node): + nodes['ass_attr'] = 'AssAttr' + + def __init__(self, expr, attrname, flags): + self.expr = expr + self.attrname = attrname + self.flags = flags + self._children = ('ass_attr', expr, attrname, flags) + + def __repr__(self): + return "AssAttr(%s,%s,%s)" % self._children[1:] + +class List(Node): + nodes['list'] = 'List' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('list', nodes) + + def __repr__(self): + return "List(%s)" % self._children[1:] + +class Dict(Node): + nodes['dict'] = 'Dict' + + def __init__(self, keys): + self.keys = keys + self._children = ('dict', keys) + + def __repr__(self): + return "Dict(%s)" % self._children[1:] + +class Not(Node): + nodes['not'] = 'Not' + + def __init__(self, expr): + self.expr = expr + self._children = ('not', expr) + + def __repr__(self): + return "Not(%s)" % self._children[1:] + +class Compare(Node): + nodes['compare'] = 'Compare' + + def __init__(self, expr, ops): + self.expr = expr + self.ops = ops + self._children = ('compare', expr, ops) + + def __repr__(self): + return "Compare(%s,%s)" % self._children[1:] + +class Name(Node): + nodes['name'] = 'Name' + + def __init__(self, name): + self.name = name + self._children = ('name', name) + + def __repr__(self): + return "Name(%s)" % self._children[1:] + +class Global(Node): + nodes['global'] = 'Global' + + def __init__(self, names): + self.names = names + self._children = ('global', names) + + def __repr__(self): + return "Global(%s)" % self._children[1:] + +class Backquote(Node): + nodes['backquote'] = 'Backquote' + + def __init__(self, node): + self.node = node + self._children = ('backquote', node) + + def __repr__(self): + return "Backquote(%s)" % self._children[1:] + +class Getattr(Node): + nodes['getattr'] = 'Getattr' + + def __init__(self, expr, attrname): + self.expr = expr + self.attrname = attrname + self._children = ('getattr', expr, attrname) + + def __repr__(self): + return "Getattr(%s,%s)" % self._children[1:] + +class CallFunc(Node): + nodes['call_func'] = 'CallFunc' + + def __init__(self, node, args): + self.node = node + self.args = args + self._children = ('call_func', node, args) + + def __repr__(self): + return "CallFunc(%s,%s)" % self._children[1:] + +class Keyword(Node): + nodes['keyword'] = 'Keyword' + + def __init__(self, name, expr): + self.name = name + self.expr = expr + self._children = ('keyword', name, expr) + + def __repr__(self): + return "Keyword(%s,%s)" % self._children[1:] + +class Subscript(Node): + nodes['subscript'] = 'Subscript' + + def __init__(self, expr, flags, subs): + self.expr = expr + self.flags = flags + self.subs = subs + self._children = ('subscript', expr, flags, subs) + + def __repr__(self): + return "Subscript(%s,%s,%s)" % self._children[1:] + +class Ellipsis(EmptyNode): + nodes['ellipsis'] = 'Ellipsis' + _children = ('ellipsis',) + def __repr__(self): + return "Ellipsis()" + +class Sliceobj(Node): + nodes['sliceobj'] = 'Sliceobj' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('sliceobj', nodes) + + def __repr__(self): + return "Sliceobj(%s)" % self._children[1:] + +class Slice(Node): + nodes['slice'] = 'Slice' + + def __init__(self, expr, flags, lower, upper): + self.expr = expr + self.flags = flags + self.lower = lower + self.upper = upper + self._children = ('slice', expr, flags, lower, upper) + + def __repr__(self): + return "Slice(%s,%s,%s,%s)" % self._children[1:] + +class Assert(Node): + nodes['assert'] = 'Assert' + + def __init__(self, expr1, expr2): + self.expr1 = expr1 + self.expr2 = expr2 + self._children = ('assert', expr1, expr2) + + def __repr__(self): + return "Assert(%s,%s)" % self._children[1:] + +class Tuple(Node): + nodes['tuple'] = 'Tuple' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('tuple', nodes) + + def __repr__(self): + return "Tuple(%s)" % self._children[1:] + +class Or(Node): + nodes['or'] = 'Or' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('or', nodes) + + def __repr__(self): + return "Or(%s)" % self._children[1:] + +class And(Node): + nodes['and'] = 'And' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('and', nodes) + + def __repr__(self): + return "And(%s)" % self._children[1:] + +class Bitor(Node): + nodes['bitor'] = 'Bitor' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitor', nodes) + + def __repr__(self): + return "Bitor(%s)" % self._children[1:] + +class Bitxor(Node): + nodes['bitxor'] = 'Bitxor' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitxor', nodes) + + def __repr__(self): + return "Bitxor(%s)" % self._children[1:] + +class Bitand(Node): + nodes['bitand'] = 'Bitand' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitand', nodes) + + def __repr__(self): + return "Bitand(%s)" % self._children[1:] + +class LeftShift(Node): + nodes['<<'] = 'LeftShift' + + def __init__(self, (expr, shift)): + self.expr = expr + self.shift = shift + self._children = ('<<', (expr, shift)) + + def __repr__(self): + return "LeftShift(%s,%s)" % self._children[1:] + +class RightShift(Node): + nodes['>>'] = 'RightShift' + + def __init__(self, (expr, shift)): + self.expr = expr + self.shift = shift + self._children = ('>>', (expr, shift)) + + def __repr__(self): + return "RightShift(%s,%s)" % self._children[1:] + +class Add(Node): + nodes['+'] = 'Add' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('+', (left, right)) + + def __repr__(self): + return "Add(%s)" % self._children[1:] + +class Sub(Node): + nodes['-'] = 'Sub' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('-', (left, right)) + + def __repr__(self): + return "Sub(%s)" % self._children[1:] + +class Mul(Node): + nodes['*'] = 'Mul' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('*', (left, right)) + + def __repr__(self): + return "Mul(%s)" % self._children[1:] + +class Div(Node): + nodes['/'] = 'Div' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('/', (left, right)) + + def __repr__(self): + return "Div(%s)" % self._children[1:] + +class Mod(Node): + nodes['%'] = 'Mod' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('%', (left, right)) + + def __repr__(self): + return "Mod(%s)" % self._children[1:] + +class Power(Node): + nodes['power'] = 'Power' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('power', (left, right)) + + def __repr__(self): + return "Power(%s)" % self._children[1:] + +class UnaryAdd(Node): + nodes['unary+'] = 'UnaryAdd' + + def __init__(self, node): + self.node = node + self._children = ('unary+', node) + + def __repr__(self): + return "UnaryAdd(%s)" % self._children[1:] + +class UnarySub(Node): + nodes['unary-'] = 'UnarySub' + + def __init__(self, node): + self.node = node + self._children = ('unary-', node) + + def __repr__(self): + return "UnarySub(%s)" % self._children[1:] + +class Invert(Node): + nodes['invert'] = 'Invert' + + def __init__(self, node): + self.node = node + self._children = ('invert', node) + + def __repr__(self): + return "Invert(%s)" % self._children[1:] + +# now clean up the nodes dictionary +klasses = globals() +for k in nodes.keys(): + nodes[k] = klasses[nodes[k]] + +# Local Variables: +# mode:python +# indent-tabs-mode: nil +# py-indent-offset: 2 +# py-smart-indentation: nil +# End: diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py new file mode 100644 index 00000000000..9200f7acbd5 --- /dev/null +++ b/Lib/compiler/transformer.py @@ -0,0 +1,1204 @@ +# Copyright 1997-1998 Greg Stein and Bill Tutt +# +# transformer.py -- transforms Python parse trees +# +# Takes an input parse tree and transforms it into a higher-level parse +# tree that is a bit more amenable to code generation. Essentially, it +# simply introduces some additional semantics. +# +# Written by Greg Stein (gstein@lyra.org) +# and Bill Tutt (rassilon@lima.mudlib.org) +# February 1997. +# +# Support for Node subclasses written by +# Jeremy Hylton (jeremy@cnri.reston.va.us) +# +# The output tree has the following nodes: +# +# Source Python line #'s appear at the end of each of all of these nodes +# If a line # doesn't apply, there will be a None instead. +# +# module: doc, node +# stmt: [ node1, ..., nodeN ] +# function: name, argnames, defaults, flags, doc, codeNode +# lambda: argnames, defaults, flags, codeNode +# classdef: name, bases, doc, codeNode +# pass: +# break: +# continue: +# for: assignNode, listNode, bodyNode, elseNode +# while: testNode, bodyNode, elseNode +# if: [ (testNode, suiteNode), ... ], elseNode +# exec: expr1Node, expr2Node, expr3Node +# from: modname, [ name1, ..., nameN ] +# import: [ name1, ..., nameN ] +# raise: expr1Node, expr2Node, expr3Node +# tryfinally: trySuiteNode, finSuiteNode +# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode +# return: valueNode +# const: value +# print: [ node1, ..., nodeN ] +# printnl: [ node1, ..., nodeN ] +# discard: exprNode +# assign: [ node1, ..., nodeN ], exprNode +# ass_tuple: [ node1, ..., nodeN ] +# ass_list: [ node1, ..., nodeN ] +# ass_name: name, flags +# ass_attr: exprNode, attrname, flags +# list: [ node1, ..., nodeN ] +# dict: [ (key1, val1), ..., (keyN, valN) ] +# not: exprNode +# compare: exprNode, [ (op, node), ..., (op, node) ] +# name: name +# global: [ name1, ..., nameN ] +# backquote: node +# getattr: exprNode, attrname +# call_func: node, [ arg1, ..., argN ] +# keyword: name, exprNode +# subscript: exprNode, flags, [ sub1, ..., subN ] +# ellipsis: +# sliceobj: [ node1, ..., nodeN ] +# slice: exprNode, flags, lowerNode, upperNode +# assert: expr1, expr2 +# +# Compiled as "binary" ops: +# tuple: [ node1, ..., nodeN ] +# or: [ node1, ..., nodeN ] +# and: [ node1, ..., nodeN ] +# bitor: [ node1, ..., nodeN ] +# bitxor: [ node1, ..., nodeN ] +# bitand: [ node1, ..., nodeN ] +# +# Operations easily evaluateable on constants: +# <<: exprNode, shiftNode +# >>: exprNode, shiftNode +# +: leftNode, rightNode +# -: leftNode, rightNode +# *: leftNode, rightNode +# /: leftNode, rightNode +# %: leftNode, rightNode +# power: leftNode, rightNode +# unary+: node +# unary-: node +# invert: node +# + +"""Parse tree transformation module. + +Exposes the Transformer class with a number of methods for returning a +"cleansed AST" instead of the parse tree that the parser exposes. +""" + +import ast +import parser +import symbol +import token +import string + +import pprint + +error = 'walker.error' + +# code flags +CO_VARARGS = 1 +CO_VARKEYWORDS = 2 + +# operation flags +OP_ASSIGN = 'OP_ASSIGN' +OP_DELETE = 'OP_DELETE' +OP_APPLY = 'OP_APPLY' + +def asList(nodes): + l = [] + for item in nodes: + if hasattr(item, "asList"): + l.append(item.asList()) + else: + if type(item) is type( (None, None) ): + l.append(tuple(asList(item))) + elif type(item) is type( [] ): + l.append(asList(item)) + else: + l.append(item) + return l + +def Node(*args): + kind = args[0] + if ast.nodes.has_key(kind): + try: + return apply(ast.nodes[kind], args[1:]) + except TypeError: + print ast.nodes[kind], len(args), args + raise + else: + raise error, "Can't find appropriate Node type." + #return apply(ast.Node, args) + +class Transformer: + """Utility object for transforming Python parse trees. + + Exposes the following methods: + tree = transform(ast_tree) + tree = parsesuite(text) + tree = parseexpr(text) + tree = parsefile(fileob | filename) + """ + + def __init__(self): + self._dispatch = { } + for value, name in symbol.sym_name.items(): + if hasattr(self, name): + self._dispatch[value] = getattr(self, name) + + def transform(self, tree): + """Transform an AST into a modified parse tree.""" + if type(tree) != type(()) and type(tree) != type([]): + tree = parser.ast2tuple(tree,1) + return self.compile_node(tree) + + def parsesuite(self, text): + """Return a modified parse tree for the given suite text.""" + # Hack for handling non-native line endings on non-DOS like OSs. + text = string.replace(text, '\x0d', '') + return self.transform(parser.suite(text)) + + def parseexpr(self, text): + """Return a modified parse tree for the given expression text.""" + return self.transform(parser.expr(text)) + + def parsefile(self, file): + """Return a modified parse tree for the contents of the given file.""" + if type(file) == type(''): + file = open(file) + return self.parsesuite(file.read()) + + # -------------------------------------------------------------- + # + # PRIVATE METHODS + # + + def compile_node(self, node): + ### emit a line-number node? + n = node[0] + if n == symbol.single_input: + return self.single_input(node[1:]) + if n == symbol.file_input: + return self.file_input(node[1:]) + if n == symbol.eval_input: + return self.eval_input(node[1:]) + if n == symbol.lambdef: + return self.lambdef(node[1:]) + if n == symbol.funcdef: + return self.funcdef(node[1:]) + if n == symbol.classdef: + return self.classdef(node[1:]) + + raise error, ('unexpected node type', n) + + def single_input(self, node): + ### do we want to do anything about being "interactive" ? + + # NEWLINE | simple_stmt | compound_stmt NEWLINE + n = node[0][0] + if n != token.NEWLINE: + return self.com_stmt(node[0]) + + return Node('pass') + + def file_input(self, nodelist): + doc = self.get_docstring(nodelist, symbol.file_input) + stmts = [ ] + for node in nodelist: + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: + self.com_append_stmt(stmts, node) + return Node('module', doc, Node('stmt', stmts)) + + def eval_input(self, nodelist): + # from the built-in function input() + ### is this sufficient? + return self.com_node(nodelist[0]) + + def funcdef(self, nodelist): + # funcdef: 'def' NAME parameters ':' suite + # parameters: '(' [varargslist] ')' + + lineno = nodelist[1][2] + name = nodelist[1][1] + args = nodelist[2][2] + + if args[0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(args[1:]) + else: + names = defaults = () + flags = 0 + doc = self.get_docstring(nodelist[4]) + + # code for function + code = self.com_node(nodelist[4]) + + n = Node('function', name, names, defaults, flags, doc, code) + n.lineno = lineno + return n + + def lambdef(self, nodelist): + # lambdef: 'lambda' [varargslist] ':' test + if nodelist[2][0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(nodelist[2][1:]) + else: + names = defaults = () + flags = 0 + + # code for lambda + code = self.com_node(nodelist[-1]) + + n = Node('lambda', names, defaults, flags, code) + n.lineno = nodelist[1][2] + return n + + def classdef(self, nodelist): + # classdef: 'class' NAME ['(' testlist ')'] ':' suite + + name = nodelist[1][1] + doc = self.get_docstring(nodelist[-1]) + if nodelist[2][0] == token.COLON: + bases = [] + else: + bases = self.com_bases(nodelist[3]) + + # code for class + code = self.com_node(nodelist[-1]) + + n = Node('classdef', name, bases, doc, code) + n.lineno = nodelist[1][2] + return n + + def stmt(self, nodelist): + return self.com_stmt(nodelist[0]) + + small_stmt = stmt + flow_stmt = stmt + compound_stmt = stmt + + def simple_stmt(self, nodelist): + # small_stmt (';' small_stmt)* [';'] NEWLINE + stmts = [ ] + for i in range(0, len(nodelist), 2): + self.com_append_stmt(stmts, nodelist[i]) + return Node('stmt', stmts) + + def parameters(self, nodelist): + raise error + + def varargslist(self, nodelist): + raise error + + def fpdef(self, nodelist): + raise error + + def fplist(self, nodelist): + raise error + + def dotted_name(self, nodelist): + raise error + + def comp_op(self, nodelist): + raise error + + def trailer(self, nodelist): + raise error + + def sliceop(self, nodelist): + raise error + + def argument(self, nodelist): + raise error + + # -------------------------------------------------------------- + # + # STATEMENT NODES (invoked by com_node()) + # + + def expr_stmt(self, nodelist): + # testlist ('=' testlist)* + exprNode = self.com_node(nodelist[-1]) + if len(nodelist) == 1: + return Node('discard', exprNode) + nodes = [ ] + for i in range(0, len(nodelist) - 2, 2): + nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) + n = Node('assign', nodes, exprNode) + n.lineno = nodelist[1][2] + return n + + def print_stmt(self, nodelist): + # print: (test ',')* [test] + items = [ ] + for i in range(1, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + if nodelist[-1][0] == token.COMMA: + n = Node('print', items) + n.lineno = nodelist[0][2] + return n + n = Node('printnl', items) + n.lineno = nodelist[0][2] + return n + + def del_stmt(self, nodelist): + return self.com_assign(nodelist[1], OP_DELETE) + + def pass_stmt(self, nodelist): + # pass: + n = Node('pass') + n.lineno = nodelist[0][2] + return n + + def break_stmt(self, nodelist): + # break: + n = Node('break') + n.lineno = nodelist[0][2] + return n + + def continue_stmt(self, nodelist): + # continue + n = Node('continue') + n.lineno = nodelist[0][2] + return n + + def return_stmt(self, nodelist): + # return: [testlist] + if len(nodelist) < 2: + n = Node('return', Node('const', None)) + n.lineno = nodelist[0][2] + return n + n = Node('return', self.com_node(nodelist[1])) + n.lineno = nodelist[0][2] + return n + + def raise_stmt(self, nodelist): + # raise: [test [',' test [',' test]]] + if len(nodelist) > 5: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + if len(nodelist) > 3: + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + if len(nodelist) > 1: + expr1 = self.com_node(nodelist[1]) + else: + expr1 = None + n = Node('raise', expr1, expr2, expr3) + n.lineno = nodelist[0][2] + return n + + def import_stmt(self, nodelist): + # import: dotted_name (',' dotted_name)* | + # from: dotted_name 'import' ('*' | NAME (',' NAME)*) + names = [ ] + if nodelist[0][1][0] == 'f': + for i in range(3, len(nodelist), 2): + # note: nodelist[i] could be (token.STAR, '*') or (token.NAME, name) + names.append(nodelist[i][1]) + n = Node('from', self.com_dotted_name(nodelist[1]), names) + n.lineno = nodelist[0][2] + return n + + for i in range(1, len(nodelist), 2): + names.append(self.com_dotted_name(nodelist[i])) + n = Node('import', names) + n.lineno = nodelist[0][2] + return n + + def global_stmt(self, nodelist): + # global: NAME (',' NAME)* + names = [ ] + for i in range(1, len(nodelist), 2): + names.append(nodelist[i][1]) + n = Node('global', names) + n.lineno = nodelist[0][2] + return n + + def exec_stmt(self, nodelist): + # exec_stmt: 'exec' expr ['in' expr [',' expr]] + expr1 = self.com_node(nodelist[1]) + if len(nodelist) >= 4: + expr2 = self.com_node(nodelist[3]) + if len(nodelist) >= 6: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + else: + expr2 = expr3 = None + + n = Node('exec', expr1, expr2, expr3) + n.lineno = nodelist[0][2] + return n + + def assert_stmt(self, nodelist): + # 'assert': test, [',' test] + expr1 = self.com_node(nodelist[1]) + if (len(nodelist) == 4): + expr2 = self.com_node(nodelist[3]) + else: + expr2 = Node('name', 'None') + n = Node('assert', expr1, expr2) + n.lineno = nodelist[0][2] + return n + + def if_stmt(self, nodelist): + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] + tests = [ ] + for i in range(0, len(nodelist) - 3, 4): + testNode = self.com_node(nodelist[i + 1]) + suiteNode = self.com_node(nodelist[i + 3]) + tests.append((testNode, suiteNode)) + + if len(nodelist) % 4 == 3: + elseNode = self.com_node(nodelist[-1]) +## elseNode.lineno = nodelist[-1][1][2] + else: + elseNode = None + n = Node('if', tests, elseNode) + n.lineno = nodelist[0][2] + return n + + def while_stmt(self, nodelist): + # 'while' test ':' suite ['else' ':' suite] + + testNode = self.com_node(nodelist[1]) + bodyNode = self.com_node(nodelist[3]) + + if len(nodelist) > 4: + elseNode = self.com_node(nodelist[6]) + else: + elseNode = None + + n = Node('while', testNode, bodyNode, elseNode) + n.lineno = nodelist[0][2] + return n + + def for_stmt(self, nodelist): + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] + + assignNode = self.com_assign(nodelist[1], OP_ASSIGN) + listNode = self.com_node(nodelist[3]) + bodyNode = self.com_node(nodelist[5]) + + if len(nodelist) > 8: + elseNode = self.com_node(nodelist[8]) + else: + elseNode = None + + n = Node('for', assignNode, listNode, bodyNode, elseNode) + n.lineno = nodelist[0][2] + return n + + def try_stmt(self, nodelist): + # 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] + # | 'try' ':' suite 'finally' ':' suite + if nodelist[3][0] != symbol.except_clause: + return self.com_try_finally(nodelist) + + return self.com_try_except(nodelist) + + def suite(self, nodelist): + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT + if len(nodelist) == 1: + return self.com_stmt(nodelist[0]) + + stmts = [ ] + for node in nodelist: + if node[0] == symbol.stmt: + self.com_append_stmt(stmts, node) + return Node('stmt', stmts) + + # -------------------------------------------------------------- + # + # EXPRESSION NODES (invoked by com_node()) + # + + def testlist(self, nodelist): + # testlist: expr (',' expr)* [','] + # exprlist: expr (',' expr)* [','] + return self.com_binary('tuple', nodelist) + + exprlist = testlist + + def test(self, nodelist): + # and_test ('or' and_test)* | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + return self.com_binary('or', nodelist) + + def and_test(self, nodelist): + # not_test ('and' not_test)* + return self.com_binary('and', nodelist) + + def not_test(self, nodelist): + # 'not' not_test | comparison + result = self.com_node(nodelist[-1]) + if len(nodelist) == 2: + n = Node('not', result) + n.lineno = nodelist[0][2] + return n + return result + + def comparison(self, nodelist): + # comparison: expr (comp_op expr)* + node = self.com_node(nodelist[0]) + if len(nodelist) == 1: + return node + + results = [ ] + for i in range(2, len(nodelist), 2): + nl = nodelist[i-1] + + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' + # | 'in' | 'not' 'in' | 'is' | 'is' 'not' + n = nl[1] + if n[0] == token.NAME: + type = n[1] + if len(nl) == 3: + if type == 'not': + type = 'notin' + else: + type = 'isnot' + else: + type = _cmp_types[n[0]] + + lineno = nl[1][2] + results.append(type, self.com_node(nodelist[i])) + + # we need a special "compare" node so that we can distinguish + # 3 < x < 5 from (3 < x) < 5 + # the two have very different semantics and results (note that the + # latter form is always true) + + n = Node('compare', node, results) + n.lineno = lineno + return n + + def expr(self, nodelist): + # xor_expr ('|' xor_expr)* + return self.com_binary('bitor', nodelist) + + def xor_expr(self, nodelist): + # xor_expr ('^' xor_expr)* + return self.com_binary('bitxor', nodelist) + + def and_expr(self, nodelist): + # xor_expr ('&' xor_expr)* + return self.com_binary('bitand', nodelist) + + def shift_expr(self, nodelist): + # shift_expr ('<<'|'>>' shift_expr)* + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.LEFTSHIFT: + node = Node('<<', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('>>', [node, right]) + node.lineno = nodelist[1][2] + return node + + def arith_expr(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.PLUS: + node = Node('+', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('-', [node, right]) + node.lineno = nodelist[1][2] + return node + + def term(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.STAR: + node = Node('*', [node, right]) + node.lineno = nodelist[1][2] + elif nodelist[i-1][0] == token.SLASH: + node = Node('/', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('%', [node, right]) + node.lineno = nodelist[1][2] + return node + + def factor(self, nodelist): + t = nodelist[0][0] + node = self.com_node(nodelist[-1]) + if t == token.PLUS: + node = Node('unary+', node) + node.lineno = nodelist[0][2] + elif t == token.MINUS: + node = Node('unary-', node) + node.lineno = nodelist[0][2] + elif t == token.TILDE: + node = Node('invert', node) + node.lineno = nodelist[0][2] + return node + + def power(self, nodelist): + # power: atom trailer* ('**' factor)* + node = self.com_node(nodelist[0]) + for i in range(1, len(nodelist)): + if nodelist[i][0] == token.DOUBLESTAR: + n = Node('power', [node, self.com_node(nodelist[i+1])]) + n.lineno = nodelist[i][2] + return n + + node = self.com_apply_trailer(node, nodelist[i]) + + return node + + def atom(self, nodelist): + t = nodelist[0][0] + if t == token.LPAR: + if nodelist[1][0] == token.RPAR: + n = Node('const', ()) + n.lineno = nodelist[0][2] + return n + return self.com_node(nodelist[1]) + + if t == token.LSQB: + if nodelist[1][0] == token.RSQB: + n = Node('const', [ ]) + n.lineno = nodelist[0][2] + return n + return self.com_list_constructor(nodelist[1]) + + if t == token.LBRACE: + if nodelist[1][0] == token.RBRACE: + return Node('const', { }) + return self.com_dictmaker(nodelist[1]) + + if t == token.BACKQUOTE: + n = Node('backquote', self.com_node(nodelist[1])) + n.lineno = nodelist[0][2] + return n + + if t == token.NUMBER: + ### need to verify this matches compile.c + k = eval(nodelist[0][1]) + n = Node('const', k) + n.lineno = nodelist[0][2] + return n + + if t == token.STRING: + ### need to verify this matches compile.c + k = '' + for node in nodelist: + k = k + eval(node[1]) + n = Node('const', k) + n.lineno = nodelist[0][2] + return n + + if t == token.NAME: + ### any processing to do? + n = Node('name', nodelist[0][1]) + n.lineno = nodelist[0][2] + return n + + raise error, "unknown node type" + + # -------------------------------------------------------------- + # + # INTERNAL PARSING UTILITIES + # + + def com_node(self, node): + # Note: compile.c has handling in com_node for del_stmt, pass_stmt, + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, + # and compound_stmt. + # We'll just dispatch them. + + # + # A ';' at the end of a line can make a NEWLINE token appear here, + # Render it harmless. (genc discards ('discard', ('const', xxxx)) Nodes) + # + if node[0] == token.NEWLINE: + return Node('discard', Node('const', None)) + + if node[0] not in _legal_node_types: + raise error, 'illegal node passed to com_node: %s' % node[0] + + return self._dispatch[node[0]](node[1:]) + + def com_arglist(self, nodelist): + # varargslist: + # (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME] + # | fpdef ['=' test] (',' fpdef ['=' test])* [','] + # | ('**'|'*' '*') NAME) + # fpdef: NAME | '(' fplist ')' + # fplist: fpdef (',' fpdef)* [','] + names = [ ] + defaults = [ ] + flags = 0 + + i = 0 + while i < len(nodelist): + node = nodelist[i] + if node[0] == token.STAR or node[0] == token.DOUBLESTAR: + if node[0] == token.STAR: + node = nodelist[i+1] + if node[0] == token.NAME: + names.append(node[1]) + flags = flags | CO_VARARGS + i = i + 3 + + if i < len(nodelist): + # should be DOUBLESTAR or STAR STAR + if nodelist[i][0] == token.DOUBLESTAR: + node = nodelist[i+1] + else: + node = nodelist[i+2] + names.append(node[1]) + flags = flags | CO_VARKEYWORDS + + break + + # fpdef: NAME | '(' fplist ')' + names.append(self.com_fpdef(node)) + + i = i + 1 + if i >= len(nodelist): + break + + if nodelist[i][0] == token.EQUAL: + defaults.append(self.com_node(nodelist[i + 1])) + i = i + 2 + elif len(defaults): + # Treat "(a=1, b)" as "(a=1, b=None)" + defaults.append(Node('const', None)) + + i = i + 1 + + return names, defaults, flags + + def com_fpdef(self, node): + # fpdef: NAME | '(' fplist ')' + if node[1][0] == token.LPAR: + return self.com_fplist(node[2]) + return node[1][1] + + def com_fplist(self, node): + # fplist: fpdef (',' fpdef)* [','] + if len(node) == 2: + return self.com_fpdef(node[1]) + list = [ ] + for i in range(1, len(node), 2): + list.append(self.com_fpdef(node[i])) + return tuple(list) + + def com_dotted_name(self, node): + # String together the dotted names and return the string + name = "" + for n in node: + if type(n) == type(()) and n[0] == 1: + name = name + n[1] + '.' + return name[:-1] + + def com_bases(self, node): + bases = [ ] + for i in range(1, len(node), 2): + bases.append(self.com_node(node[i])) + return bases + + def com_try_finally(self, nodelist): + # try_fin_stmt: "try" ":" suite "finally" ":" suite + n = Node('tryfinally', self.com_node(nodelist[2]), self.com_node(nodelist[5])) + n.lineno = nodelist[0][2] + return n + + def com_try_except(self, nodelist): + # try_except: 'try' ':' suite (except_clause ':' suite)* ['else' suite] + #tryexcept: [TryNode, [except_clauses], elseNode)] + stmt = self.com_node(nodelist[2]) + clauses = [] + elseNode = None + for i in range(3, len(nodelist), 3): + node = nodelist[i] + if node[0] == symbol.except_clause: + # except_clause: 'except' [expr [',' expr]] */ + if len(node) > 2: + expr1 = self.com_node(node[2]) + if len(node) > 4: + expr2 = self.com_assign(node[4], OP_ASSIGN) + else: + expr2 = None + else: + expr1 = expr2 = None + clauses.append(expr1, expr2, self.com_node(nodelist[i+2])) + + if node[0] == token.NAME: + elseNode = self.com_node(nodelist[i+2]) + n = Node('tryexcept', self.com_node(nodelist[2]), clauses, elseNode) + n.lineno = nodelist[0][2] + return n + + def com_assign(self, node, assigning): + # return a node suitable for use as an "lvalue" + # loop to avoid trivial recursion + while 1: + t = node[0] + if t == symbol.exprlist or t == symbol.testlist: + if len(node) > 2: + return self.com_assign_tuple(node, assigning) + node = node[1] + elif t in _assign_types: + if len(node) > 2: + raise SyntaxError, "can't assign to operator" + node = node[1] + elif t == symbol.power: + if node[1][0] != symbol.atom: + raise SyntaxError, "can't assign to operator" + if len(node) > 2: + primary = self.com_node(node[1]) + for i in range(2, len(node)-1): + ch = node[i] + if ch[0] == token.DOUBLESTAR: + raise SyntaxError, "can't assign to operator" + primary = self.com_apply_trailer(primary, ch) + return self.com_assign_trailer(primary, node[-1], assigning) + node = node[1] + elif t == symbol.atom: + t = node[1][0] + if t == token.LPAR: + node = node[2] + if node[0] == token.RPAR: + raise SyntaxError, "can't assign to ()" + elif t == token.LSQB: + node = node[2] + if node[0] == token.RSQB: + raise SyntaxError, "can't assign to []" + return self.com_assign_list(node, assigning) + elif t == token.NAME: + return self.com_assign_name(node[1], assigning) + else: + raise SyntaxError, "can't assign to literal" + else: + raise SyntaxError, "bad assignment" + + def com_assign_tuple(self, node, assigning): + assigns = [ ] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return Node('ass_tuple', assigns) + + def com_assign_list(self, node, assigning): + assigns = [ ] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return Node('ass_list', assigns) + + def com_assign_name(self, node, assigning): + return Node('ass_name', node[1], assigning) + + def com_assign_trailer(self, primary, node, assigning): + t = node[1][0] + if t == token.LPAR: + raise SyntaxError, "can't assign to function call" + if t == token.DOT: + return self.com_assign_attr(primary, node[2], assigning) + if t == token.LSQB: + return self.com_subscriptlist(primary, node[2], assigning) + raise SyntaxError, "unknown trailer type: %s" % t + + def com_assign_attr(self, primary, node, assigning): + return Node('ass_attr', primary, node[1], assigning) + + def com_binary(self, type, nodelist): + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." + if len(nodelist) == 1: + return self.com_node(nodelist[0]) + items = [ ] + for i in range(0, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + return Node(type, items) + + def com_stmt(self, node): + #pprint.pprint(node) + result = self.com_node(node) + try: + result[0] + except: + print node[0] + if result[0] == 'stmt': + return result + return Node('stmt', [ result ]) + + def com_append_stmt(self, stmts, node): + result = self.com_node(node) + try: + result[0] + except: + print node + if result[0] == 'stmt': + stmts[len(stmts):] = result[1] + else: + stmts.append(result) + + def com_list_constructor(self, nodelist): + values = [ ] + for i in range(1, len(nodelist), 2): + values.append(self.com_node(nodelist[i])) + return Node('list', values) + + def com_dictmaker(self, nodelist): + # dictmaker: test ':' test (',' test ':' value)* [','] + items = [ ] + for i in range(1, len(nodelist), 4): + items.append(self.com_node(nodelist[i]), self.com_node(nodelist[i+2])) + return Node('dict', items) + + def com_apply_trailer(self, primaryNode, nodelist): + t = nodelist[1][0] + if t == token.LPAR: + return self.com_call_function(primaryNode, nodelist[2]) + if t == token.DOT: + return self.com_select_member(primaryNode, nodelist[2]) + if t == token.LSQB: + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) + + raise SyntaxError, 'unknown node type: %s' % t + + def com_select_member(self, primaryNode, nodelist): + if nodelist[0] != token.NAME: + raise SyntaxError, "member must be a name" + n = Node('getattr', primaryNode, nodelist[1]) + n.lineno = nodelist[2] + return n + + def com_call_function(self, primaryNode, nodelist): + if nodelist[0] == token.RPAR: + return Node('call_func', primaryNode, [ ]) + args = [ ] + kw = 0 + for i in range(1, len(nodelist), 2): + kw, result = self.com_argument(nodelist[i], kw) + args.append(result) + return Node('call_func', primaryNode, args) + + def com_argument(self, nodelist, kw): + if len(nodelist) == 2: + if kw: + raise SyntaxError, "non-keyword arg after keyword arg" + return 0, self.com_node(nodelist[1]) + result = self.com_node(nodelist[3]) + n = nodelist[1] + while len(n) == 2 and n[0] != token.NAME: + n = n[1] + if n[0] != token.NAME: + raise SyntaxError, "keyword can't be an expression (%s)"%n[0] + n = Node('keyword', n[1], result) + n.lineno = result.lineno + return 1, n + + def com_subscriptlist(self, primary, nodelist, assigning): + # slicing: simple_slicing | extended_slicing + # simple_slicing: primary "[" short_slice "]" + # extended_slicing: primary "[" slice_list "]" + # slice_list: slice_item ("," slice_item)* [","] + + # backwards compat slice for '[i:j]' + if len(nodelist) == 2: + sub = nodelist[1] + if (sub[1][0] == token.COLON or \ + (len(sub) > 2 and sub[2][0] == token.COLON)) and \ + sub[-1][0] != symbol.sliceop: + return self.com_slice(primary, sub, assigning) + + subscripts = [ ] + for i in range(1, len(nodelist), 2): + subscripts.append(self.com_subscript(nodelist[i])) + + return Node('subscript', primary, assigning, subscripts) + + def com_subscript(self, node): + # slice_item: expression | proper_slice | ellipsis + ch = node[1] + if ch[0] == token.DOT and node[2][0] == token.DOT: + return ('ellipsis', None) + if ch[0] == token.COLON or len(node) > 2: + return self.com_sliceobj(node) + return self.com_node(ch) + + def com_sliceobj(self, node): + # proper_slice: short_slice | long_slice + # short_slice: [lower_bound] ":" [upper_bound] + # long_slice: short_slice ":" [stride] + # lower_bound: expression + # upper_bound: expression + # stride: expression + # + # Note: a stride may be further slicing... + + items = [ ] + + if node[1][0] == token.COLON: + items.append(Node('const', None)) + i = 2 + else: + items.append(self.com_node(node[1])) + # i == 2 is a COLON + i = 3 + + if i < len(node) and node[i][0] == symbol.test: + items.append(self.com_node(node[i])) + i = i + 1 + else: + items.append(Node('const', None)) + + # a short_slice has been built. look for long_slice now by looking + # for strides... + for j in range(i, len(node)): + ch = node[j] + if len(ch) == 2: + items.append(Node('const', None)) + else: + items.append(self.com_node(ch[2])) + + return Node('sliceobj', items) + + def com_slice(self, primary, node, assigning): + # short_slice: [lower_bound] ":" [upper_bound] + lower = upper = None + if len(node) == 3: + if node[1][0] == token.COLON: + upper = self.com_node(node[2]) + else: + lower = self.com_node(node[1]) + elif len(node) == 4: + lower = self.com_node(node[1]) + upper = self.com_node(node[3]) + return Node('slice', primary, assigning, lower, upper) + + def get_docstring(self, node, n=None): + if n is None: + n = node[0] + node = node[1:] + if n == symbol.suite: + if len(node) == 1: + return self.get_docstring(node[0]) + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.file_input: + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.atom: + if node[0][0] == token.STRING: + s = '' + for t in node: + s = s + eval(t[1]) + return s + return None + if n == symbol.stmt or n == symbol.simple_stmt or n == symbol.small_stmt: + return self.get_docstring(node[0]) + if n in _doc_nodes and len(node) == 1: + return self.get_docstring(node[0]) + return None + + +_doc_nodes = [ + symbol.expr_stmt, + symbol.testlist, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + ] + +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' +# | 'in' | 'not' 'in' | 'is' | 'is' 'not' +_cmp_types = { + token.LESS : '<', + token.GREATER : '>', + token.EQEQUAL : '==', + token.EQUAL : '==', + token.LESSEQUAL : '<=', + token.GREATEREQUAL : '>=', + token.NOTEQUAL : '!=', + } + +_legal_node_types = [ + symbol.funcdef, + symbol.classdef, + symbol.stmt, + symbol.small_stmt, + symbol.flow_stmt, + symbol.simple_stmt, + symbol.compound_stmt, + symbol.expr_stmt, + symbol.print_stmt, + symbol.del_stmt, + symbol.pass_stmt, + symbol.break_stmt, + symbol.continue_stmt, + symbol.return_stmt, + symbol.raise_stmt, + symbol.import_stmt, + symbol.global_stmt, + symbol.exec_stmt, + symbol.assert_stmt, + symbol.if_stmt, + symbol.while_stmt, + symbol.for_stmt, + symbol.try_stmt, + symbol.suite, + symbol.testlist, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.exprlist, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + symbol.atom, + ] + +_assign_types = [ + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + ] + +# Local Variables: +# mode: python +# indent-tabs-mode: nil +# py-indent-offset: 2 +# py-smart-indentation: nil +# End: + diff --git a/Tools/compiler/compiler/ast.py b/Tools/compiler/compiler/ast.py new file mode 100644 index 00000000000..9c214d453d1 --- /dev/null +++ b/Tools/compiler/compiler/ast.py @@ -0,0 +1,689 @@ +import types + +nodes = {} + +def flatten(list): + l = [] + for elt in list: + if type(elt) is types.TupleType: + for elt2 in flatten(elt): + l.append(elt2) + elif type(elt) is types.ListType: + for elt2 in flatten(elt): + l.append(elt2) + else: + l.append(elt) + return l + +def asList(nodes): + l = [] + for item in nodes: + if hasattr(item, "asList"): + l.append(item.asList()) + else: + if type(item) is types.TupleType: + l.append(tuple(asList(item))) + elif type(item) is types.ListType: + l.append(asList(item)) + else: + l.append(item) + return l + +class Node: + def __init__(self, *args): + self._children = args + self.lineno = None + def __getitem__(self, index): + return self._children[index] + def __repr__(self): + return "" % self._children[0] + def __len__(self): + return len(self._children) + def __getslice__(self, low, high): + return self._children[low:high] + def getChildren(self): + return tuple(flatten(self._children[1:])) + def getType(self): + return self._children[0] + def asList(self): + return tuple(asList(self._children)) + +class EmptyNode(Node): + def __init__(self): + self.lineno = None + +class Module(Node): + nodes['module'] = 'Module' + + def __init__(self, doc, node): + self.doc = doc + self.node = node + self._children = ('module', doc, node) + + def __repr__(self): + return "Module(%s,%s)" % self._children[1:] + +class Stmt(Node): + nodes['stmt'] = 'Stmt' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('stmt', nodes) + + def __repr__(self): + return "Stmt(%s)" % self._children[1:] + +class Function(Node): + nodes['function'] = 'Function' + + def __init__(self, name, argnames, defaults, flags, doc, code): + self.name = name + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.doc = doc + self.code = code + self._children = ('function', + name, argnames, defaults, flags, doc, code) + + def __repr__(self): + return "Function(%s,%s,%s,%s,%s,%s)" % self._children[1:] + +class Lambda(Node): + nodes['lambda'] = 'Lambda' + + def __init__(self, argnames, defaults, flags, code): + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.code = code + self._children = ('lambda', argnames, defaults, flags, code) + + def __repr__(self): + return "Lambda(%s,%s,%s,%s)" % self._children[1:] + +class Classdef(Node): + nodes['classdef'] = 'Classdef' + + def __init__(self, name, bases, doc, code): + self.name = name + self.bases = bases + self.doc = doc + self.code = code + self._children = ('classdef', name, bases, doc, code) + + def __repr__(self): + return "Classdef(%s,%s,%s,%s)" % self._children[1:] + +class Pass(EmptyNode): + nodes['pass'] = 'Pass' + _children = ('pass',) + def __repr__(self): + return "Pass()" + +class Break(EmptyNode): + nodes['break'] = 'Break' + _children = ('break',) + def __repr__(self): + return "Break()" + +class Continue(EmptyNode): + nodes['continue'] = 'Continue' + _children = ('continue',) + def __repr__(self): + return "Continue()" + +class For(Node): + nodes['for'] = 'For' + + def __init__(self, assign, list, body, else_): + self.assign = assign + self.list = list + self.body = body + self.else_ = else_ + self._children = ('for', assign, list, body, else_) + + def __repr__(self): + return "For(%s,%s,%s,%s)" % self._children[1:] + +class While(Node): + nodes['while'] = 'While' + + def __init__(self, test, body, else_): + self.test = test + self.body = body + self.else_ = else_ + self._children = ('while', test, body, else_) + + def __repr__(self): + return "While(%s,%s,%s)" % self._children[1:] + +class If(Node): + """if: [ (testNode, suiteNode), ...], elseNode""" + nodes['if'] = 'If' + + def __init__(self, tests, else_): + self.tests = tests + self.else_ = else_ + self._children = ('if', tests, else_) + + def __repr__(self): + return "If(%s,%s)" % self._children[1:] + +class Exec(Node): + nodes['exec'] = 'Exec' + + def __init__(self, expr1, expr2, expr3): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self._children = ('exec', expr1, expr2, expr3) + + def __repr__(self): + return "Exec(%s,%s,%s)" % self._children[1:] + +class From(Node): + nodes['from'] = 'From' + + def __init__(self, modname, names): + self.modname = modname + self.names = names + self._children = ('from', modname, names) + + def __repr__(self): + return "From(%s,%s)" % self._children[1:] + +class Import(Node): + nodes['import'] = 'Import' + + def __init__(self, names): + self.names = names + self._children = ('import', names) + + def __repr__(self): + return "Import(%s)" % self._children[1:] + +class Raise(Node): + nodes['raise'] = 'Raise' + + def __init__(self, expr1, expr2, expr3): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self._children = ('raise', expr1, expr2, expr3) + + def __repr__(self): + return "Raise(%s,%s,%s)" % self._children[1:] + +class Tryfinally(Node): + nodes['tryfinally'] = 'Tryfinally' + + def __init__(self, try_, fin): + self.try_ = try_ + self.fin = fin + self._children = ('tryfinally', try_, fin) + + def __repr__(self): + return "Tryfinally(%s,%s)" % self._children[1:] + +class Tryexcept(Node): + nodes['tryexcept'] = 'Tryexcept' + + def __init__(self, try_, excs, else_): + self.try_ = try_ + self.excs = excs + self.else_ = else_ + self._children = ('tryexcept', try_, excs, else_) + + def __repr__(self): + return "Tryexcept(%s,%s,%s)" % self._children[1:] + +class Return(Node): + nodes['return'] = 'Return' + + def __init__(self, value): + self.value = value + self._children = ('return', value) + + def __repr__(self): + return "Return(%s)" % self._children[1:] + +class Const(Node): + nodes['const'] = 'Const' + + def __init__(self, value): + self.value = value + self._children = ('const', value) + + def __repr__(self): + return "Const(%s)" % self._children[1:] + +class Print(Node): + nodes['print'] = 'Print' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('print', nodes) + + def __repr__(self): + return "Print(%s)" % self._children[1:] + +class Printnl(Node): + nodes['printnl'] = 'Printnl' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('printnl', nodes) + + def __repr__(self): + return "Printnl(%s)" % self._children[1:] + +class Discard(Node): + nodes['discard'] = 'Discard' + + def __init__(self, expr): + self.expr = expr + self._children = ('discard', expr) + + def __repr__(self): + return "Discard(%s)" % self._children[1:] + +class Assign(Node): + nodes['assign'] = 'Assign' + + def __init__(self, nodes, expr): + self.nodes = nodes + self.expr = expr + self._children = ('assign', nodes, expr) + + def __repr__(self): + return "Assign(%s,%s)" % self._children[1:] + +class AssTuple(Node): + nodes['ass_tuple'] = 'AssTuple' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('ass_tuple', nodes) + + def __repr__(self): + return "AssTuple(%s)" % self._children[1:] + +class AssList(Node): + nodes['ass_list'] = 'AssList' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('ass_list', nodes) + + def __repr__(self): + return "AssList(%s)" % self._children[1:] + +class AssName(Node): + nodes['ass_name'] = 'AssName' + + def __init__(self, name, flags): + self.name = name + self.flags = flags + self._children = ('ass_name', name, flags) + + def __repr__(self): + return "AssName(%s,%s)" % self._children[1:] + +class AssAttr(Node): + nodes['ass_attr'] = 'AssAttr' + + def __init__(self, expr, attrname, flags): + self.expr = expr + self.attrname = attrname + self.flags = flags + self._children = ('ass_attr', expr, attrname, flags) + + def __repr__(self): + return "AssAttr(%s,%s,%s)" % self._children[1:] + +class List(Node): + nodes['list'] = 'List' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('list', nodes) + + def __repr__(self): + return "List(%s)" % self._children[1:] + +class Dict(Node): + nodes['dict'] = 'Dict' + + def __init__(self, keys): + self.keys = keys + self._children = ('dict', keys) + + def __repr__(self): + return "Dict(%s)" % self._children[1:] + +class Not(Node): + nodes['not'] = 'Not' + + def __init__(self, expr): + self.expr = expr + self._children = ('not', expr) + + def __repr__(self): + return "Not(%s)" % self._children[1:] + +class Compare(Node): + nodes['compare'] = 'Compare' + + def __init__(self, expr, ops): + self.expr = expr + self.ops = ops + self._children = ('compare', expr, ops) + + def __repr__(self): + return "Compare(%s,%s)" % self._children[1:] + +class Name(Node): + nodes['name'] = 'Name' + + def __init__(self, name): + self.name = name + self._children = ('name', name) + + def __repr__(self): + return "Name(%s)" % self._children[1:] + +class Global(Node): + nodes['global'] = 'Global' + + def __init__(self, names): + self.names = names + self._children = ('global', names) + + def __repr__(self): + return "Global(%s)" % self._children[1:] + +class Backquote(Node): + nodes['backquote'] = 'Backquote' + + def __init__(self, node): + self.node = node + self._children = ('backquote', node) + + def __repr__(self): + return "Backquote(%s)" % self._children[1:] + +class Getattr(Node): + nodes['getattr'] = 'Getattr' + + def __init__(self, expr, attrname): + self.expr = expr + self.attrname = attrname + self._children = ('getattr', expr, attrname) + + def __repr__(self): + return "Getattr(%s,%s)" % self._children[1:] + +class CallFunc(Node): + nodes['call_func'] = 'CallFunc' + + def __init__(self, node, args): + self.node = node + self.args = args + self._children = ('call_func', node, args) + + def __repr__(self): + return "CallFunc(%s,%s)" % self._children[1:] + +class Keyword(Node): + nodes['keyword'] = 'Keyword' + + def __init__(self, name, expr): + self.name = name + self.expr = expr + self._children = ('keyword', name, expr) + + def __repr__(self): + return "Keyword(%s,%s)" % self._children[1:] + +class Subscript(Node): + nodes['subscript'] = 'Subscript' + + def __init__(self, expr, flags, subs): + self.expr = expr + self.flags = flags + self.subs = subs + self._children = ('subscript', expr, flags, subs) + + def __repr__(self): + return "Subscript(%s,%s,%s)" % self._children[1:] + +class Ellipsis(EmptyNode): + nodes['ellipsis'] = 'Ellipsis' + _children = ('ellipsis',) + def __repr__(self): + return "Ellipsis()" + +class Sliceobj(Node): + nodes['sliceobj'] = 'Sliceobj' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('sliceobj', nodes) + + def __repr__(self): + return "Sliceobj(%s)" % self._children[1:] + +class Slice(Node): + nodes['slice'] = 'Slice' + + def __init__(self, expr, flags, lower, upper): + self.expr = expr + self.flags = flags + self.lower = lower + self.upper = upper + self._children = ('slice', expr, flags, lower, upper) + + def __repr__(self): + return "Slice(%s,%s,%s,%s)" % self._children[1:] + +class Assert(Node): + nodes['assert'] = 'Assert' + + def __init__(self, expr1, expr2): + self.expr1 = expr1 + self.expr2 = expr2 + self._children = ('assert', expr1, expr2) + + def __repr__(self): + return "Assert(%s,%s)" % self._children[1:] + +class Tuple(Node): + nodes['tuple'] = 'Tuple' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('tuple', nodes) + + def __repr__(self): + return "Tuple(%s)" % self._children[1:] + +class Or(Node): + nodes['or'] = 'Or' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('or', nodes) + + def __repr__(self): + return "Or(%s)" % self._children[1:] + +class And(Node): + nodes['and'] = 'And' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('and', nodes) + + def __repr__(self): + return "And(%s)" % self._children[1:] + +class Bitor(Node): + nodes['bitor'] = 'Bitor' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitor', nodes) + + def __repr__(self): + return "Bitor(%s)" % self._children[1:] + +class Bitxor(Node): + nodes['bitxor'] = 'Bitxor' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitxor', nodes) + + def __repr__(self): + return "Bitxor(%s)" % self._children[1:] + +class Bitand(Node): + nodes['bitand'] = 'Bitand' + + def __init__(self, nodes): + self.nodes = nodes + self._children = ('bitand', nodes) + + def __repr__(self): + return "Bitand(%s)" % self._children[1:] + +class LeftShift(Node): + nodes['<<'] = 'LeftShift' + + def __init__(self, (expr, shift)): + self.expr = expr + self.shift = shift + self._children = ('<<', (expr, shift)) + + def __repr__(self): + return "LeftShift(%s,%s)" % self._children[1:] + +class RightShift(Node): + nodes['>>'] = 'RightShift' + + def __init__(self, (expr, shift)): + self.expr = expr + self.shift = shift + self._children = ('>>', (expr, shift)) + + def __repr__(self): + return "RightShift(%s,%s)" % self._children[1:] + +class Add(Node): + nodes['+'] = 'Add' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('+', (left, right)) + + def __repr__(self): + return "Add(%s)" % self._children[1:] + +class Sub(Node): + nodes['-'] = 'Sub' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('-', (left, right)) + + def __repr__(self): + return "Sub(%s)" % self._children[1:] + +class Mul(Node): + nodes['*'] = 'Mul' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('*', (left, right)) + + def __repr__(self): + return "Mul(%s)" % self._children[1:] + +class Div(Node): + nodes['/'] = 'Div' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('/', (left, right)) + + def __repr__(self): + return "Div(%s)" % self._children[1:] + +class Mod(Node): + nodes['%'] = 'Mod' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('%', (left, right)) + + def __repr__(self): + return "Mod(%s)" % self._children[1:] + +class Power(Node): + nodes['power'] = 'Power' + + def __init__(self, (left, right)): + self.left = left + self.right = right + self._children = ('power', (left, right)) + + def __repr__(self): + return "Power(%s)" % self._children[1:] + +class UnaryAdd(Node): + nodes['unary+'] = 'UnaryAdd' + + def __init__(self, node): + self.node = node + self._children = ('unary+', node) + + def __repr__(self): + return "UnaryAdd(%s)" % self._children[1:] + +class UnarySub(Node): + nodes['unary-'] = 'UnarySub' + + def __init__(self, node): + self.node = node + self._children = ('unary-', node) + + def __repr__(self): + return "UnarySub(%s)" % self._children[1:] + +class Invert(Node): + nodes['invert'] = 'Invert' + + def __init__(self, node): + self.node = node + self._children = ('invert', node) + + def __repr__(self): + return "Invert(%s)" % self._children[1:] + +# now clean up the nodes dictionary +klasses = globals() +for k in nodes.keys(): + nodes[k] = klasses[nodes[k]] + +# Local Variables: +# mode:python +# indent-tabs-mode: nil +# py-indent-offset: 2 +# py-smart-indentation: nil +# End: diff --git a/Tools/compiler/compiler/transformer.py b/Tools/compiler/compiler/transformer.py new file mode 100644 index 00000000000..9200f7acbd5 --- /dev/null +++ b/Tools/compiler/compiler/transformer.py @@ -0,0 +1,1204 @@ +# Copyright 1997-1998 Greg Stein and Bill Tutt +# +# transformer.py -- transforms Python parse trees +# +# Takes an input parse tree and transforms it into a higher-level parse +# tree that is a bit more amenable to code generation. Essentially, it +# simply introduces some additional semantics. +# +# Written by Greg Stein (gstein@lyra.org) +# and Bill Tutt (rassilon@lima.mudlib.org) +# February 1997. +# +# Support for Node subclasses written by +# Jeremy Hylton (jeremy@cnri.reston.va.us) +# +# The output tree has the following nodes: +# +# Source Python line #'s appear at the end of each of all of these nodes +# If a line # doesn't apply, there will be a None instead. +# +# module: doc, node +# stmt: [ node1, ..., nodeN ] +# function: name, argnames, defaults, flags, doc, codeNode +# lambda: argnames, defaults, flags, codeNode +# classdef: name, bases, doc, codeNode +# pass: +# break: +# continue: +# for: assignNode, listNode, bodyNode, elseNode +# while: testNode, bodyNode, elseNode +# if: [ (testNode, suiteNode), ... ], elseNode +# exec: expr1Node, expr2Node, expr3Node +# from: modname, [ name1, ..., nameN ] +# import: [ name1, ..., nameN ] +# raise: expr1Node, expr2Node, expr3Node +# tryfinally: trySuiteNode, finSuiteNode +# tryexcept: trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode +# return: valueNode +# const: value +# print: [ node1, ..., nodeN ] +# printnl: [ node1, ..., nodeN ] +# discard: exprNode +# assign: [ node1, ..., nodeN ], exprNode +# ass_tuple: [ node1, ..., nodeN ] +# ass_list: [ node1, ..., nodeN ] +# ass_name: name, flags +# ass_attr: exprNode, attrname, flags +# list: [ node1, ..., nodeN ] +# dict: [ (key1, val1), ..., (keyN, valN) ] +# not: exprNode +# compare: exprNode, [ (op, node), ..., (op, node) ] +# name: name +# global: [ name1, ..., nameN ] +# backquote: node +# getattr: exprNode, attrname +# call_func: node, [ arg1, ..., argN ] +# keyword: name, exprNode +# subscript: exprNode, flags, [ sub1, ..., subN ] +# ellipsis: +# sliceobj: [ node1, ..., nodeN ] +# slice: exprNode, flags, lowerNode, upperNode +# assert: expr1, expr2 +# +# Compiled as "binary" ops: +# tuple: [ node1, ..., nodeN ] +# or: [ node1, ..., nodeN ] +# and: [ node1, ..., nodeN ] +# bitor: [ node1, ..., nodeN ] +# bitxor: [ node1, ..., nodeN ] +# bitand: [ node1, ..., nodeN ] +# +# Operations easily evaluateable on constants: +# <<: exprNode, shiftNode +# >>: exprNode, shiftNode +# +: leftNode, rightNode +# -: leftNode, rightNode +# *: leftNode, rightNode +# /: leftNode, rightNode +# %: leftNode, rightNode +# power: leftNode, rightNode +# unary+: node +# unary-: node +# invert: node +# + +"""Parse tree transformation module. + +Exposes the Transformer class with a number of methods for returning a +"cleansed AST" instead of the parse tree that the parser exposes. +""" + +import ast +import parser +import symbol +import token +import string + +import pprint + +error = 'walker.error' + +# code flags +CO_VARARGS = 1 +CO_VARKEYWORDS = 2 + +# operation flags +OP_ASSIGN = 'OP_ASSIGN' +OP_DELETE = 'OP_DELETE' +OP_APPLY = 'OP_APPLY' + +def asList(nodes): + l = [] + for item in nodes: + if hasattr(item, "asList"): + l.append(item.asList()) + else: + if type(item) is type( (None, None) ): + l.append(tuple(asList(item))) + elif type(item) is type( [] ): + l.append(asList(item)) + else: + l.append(item) + return l + +def Node(*args): + kind = args[0] + if ast.nodes.has_key(kind): + try: + return apply(ast.nodes[kind], args[1:]) + except TypeError: + print ast.nodes[kind], len(args), args + raise + else: + raise error, "Can't find appropriate Node type." + #return apply(ast.Node, args) + +class Transformer: + """Utility object for transforming Python parse trees. + + Exposes the following methods: + tree = transform(ast_tree) + tree = parsesuite(text) + tree = parseexpr(text) + tree = parsefile(fileob | filename) + """ + + def __init__(self): + self._dispatch = { } + for value, name in symbol.sym_name.items(): + if hasattr(self, name): + self._dispatch[value] = getattr(self, name) + + def transform(self, tree): + """Transform an AST into a modified parse tree.""" + if type(tree) != type(()) and type(tree) != type([]): + tree = parser.ast2tuple(tree,1) + return self.compile_node(tree) + + def parsesuite(self, text): + """Return a modified parse tree for the given suite text.""" + # Hack for handling non-native line endings on non-DOS like OSs. + text = string.replace(text, '\x0d', '') + return self.transform(parser.suite(text)) + + def parseexpr(self, text): + """Return a modified parse tree for the given expression text.""" + return self.transform(parser.expr(text)) + + def parsefile(self, file): + """Return a modified parse tree for the contents of the given file.""" + if type(file) == type(''): + file = open(file) + return self.parsesuite(file.read()) + + # -------------------------------------------------------------- + # + # PRIVATE METHODS + # + + def compile_node(self, node): + ### emit a line-number node? + n = node[0] + if n == symbol.single_input: + return self.single_input(node[1:]) + if n == symbol.file_input: + return self.file_input(node[1:]) + if n == symbol.eval_input: + return self.eval_input(node[1:]) + if n == symbol.lambdef: + return self.lambdef(node[1:]) + if n == symbol.funcdef: + return self.funcdef(node[1:]) + if n == symbol.classdef: + return self.classdef(node[1:]) + + raise error, ('unexpected node type', n) + + def single_input(self, node): + ### do we want to do anything about being "interactive" ? + + # NEWLINE | simple_stmt | compound_stmt NEWLINE + n = node[0][0] + if n != token.NEWLINE: + return self.com_stmt(node[0]) + + return Node('pass') + + def file_input(self, nodelist): + doc = self.get_docstring(nodelist, symbol.file_input) + stmts = [ ] + for node in nodelist: + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: + self.com_append_stmt(stmts, node) + return Node('module', doc, Node('stmt', stmts)) + + def eval_input(self, nodelist): + # from the built-in function input() + ### is this sufficient? + return self.com_node(nodelist[0]) + + def funcdef(self, nodelist): + # funcdef: 'def' NAME parameters ':' suite + # parameters: '(' [varargslist] ')' + + lineno = nodelist[1][2] + name = nodelist[1][1] + args = nodelist[2][2] + + if args[0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(args[1:]) + else: + names = defaults = () + flags = 0 + doc = self.get_docstring(nodelist[4]) + + # code for function + code = self.com_node(nodelist[4]) + + n = Node('function', name, names, defaults, flags, doc, code) + n.lineno = lineno + return n + + def lambdef(self, nodelist): + # lambdef: 'lambda' [varargslist] ':' test + if nodelist[2][0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(nodelist[2][1:]) + else: + names = defaults = () + flags = 0 + + # code for lambda + code = self.com_node(nodelist[-1]) + + n = Node('lambda', names, defaults, flags, code) + n.lineno = nodelist[1][2] + return n + + def classdef(self, nodelist): + # classdef: 'class' NAME ['(' testlist ')'] ':' suite + + name = nodelist[1][1] + doc = self.get_docstring(nodelist[-1]) + if nodelist[2][0] == token.COLON: + bases = [] + else: + bases = self.com_bases(nodelist[3]) + + # code for class + code = self.com_node(nodelist[-1]) + + n = Node('classdef', name, bases, doc, code) + n.lineno = nodelist[1][2] + return n + + def stmt(self, nodelist): + return self.com_stmt(nodelist[0]) + + small_stmt = stmt + flow_stmt = stmt + compound_stmt = stmt + + def simple_stmt(self, nodelist): + # small_stmt (';' small_stmt)* [';'] NEWLINE + stmts = [ ] + for i in range(0, len(nodelist), 2): + self.com_append_stmt(stmts, nodelist[i]) + return Node('stmt', stmts) + + def parameters(self, nodelist): + raise error + + def varargslist(self, nodelist): + raise error + + def fpdef(self, nodelist): + raise error + + def fplist(self, nodelist): + raise error + + def dotted_name(self, nodelist): + raise error + + def comp_op(self, nodelist): + raise error + + def trailer(self, nodelist): + raise error + + def sliceop(self, nodelist): + raise error + + def argument(self, nodelist): + raise error + + # -------------------------------------------------------------- + # + # STATEMENT NODES (invoked by com_node()) + # + + def expr_stmt(self, nodelist): + # testlist ('=' testlist)* + exprNode = self.com_node(nodelist[-1]) + if len(nodelist) == 1: + return Node('discard', exprNode) + nodes = [ ] + for i in range(0, len(nodelist) - 2, 2): + nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) + n = Node('assign', nodes, exprNode) + n.lineno = nodelist[1][2] + return n + + def print_stmt(self, nodelist): + # print: (test ',')* [test] + items = [ ] + for i in range(1, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + if nodelist[-1][0] == token.COMMA: + n = Node('print', items) + n.lineno = nodelist[0][2] + return n + n = Node('printnl', items) + n.lineno = nodelist[0][2] + return n + + def del_stmt(self, nodelist): + return self.com_assign(nodelist[1], OP_DELETE) + + def pass_stmt(self, nodelist): + # pass: + n = Node('pass') + n.lineno = nodelist[0][2] + return n + + def break_stmt(self, nodelist): + # break: + n = Node('break') + n.lineno = nodelist[0][2] + return n + + def continue_stmt(self, nodelist): + # continue + n = Node('continue') + n.lineno = nodelist[0][2] + return n + + def return_stmt(self, nodelist): + # return: [testlist] + if len(nodelist) < 2: + n = Node('return', Node('const', None)) + n.lineno = nodelist[0][2] + return n + n = Node('return', self.com_node(nodelist[1])) + n.lineno = nodelist[0][2] + return n + + def raise_stmt(self, nodelist): + # raise: [test [',' test [',' test]]] + if len(nodelist) > 5: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + if len(nodelist) > 3: + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + if len(nodelist) > 1: + expr1 = self.com_node(nodelist[1]) + else: + expr1 = None + n = Node('raise', expr1, expr2, expr3) + n.lineno = nodelist[0][2] + return n + + def import_stmt(self, nodelist): + # import: dotted_name (',' dotted_name)* | + # from: dotted_name 'import' ('*' | NAME (',' NAME)*) + names = [ ] + if nodelist[0][1][0] == 'f': + for i in range(3, len(nodelist), 2): + # note: nodelist[i] could be (token.STAR, '*') or (token.NAME, name) + names.append(nodelist[i][1]) + n = Node('from', self.com_dotted_name(nodelist[1]), names) + n.lineno = nodelist[0][2] + return n + + for i in range(1, len(nodelist), 2): + names.append(self.com_dotted_name(nodelist[i])) + n = Node('import', names) + n.lineno = nodelist[0][2] + return n + + def global_stmt(self, nodelist): + # global: NAME (',' NAME)* + names = [ ] + for i in range(1, len(nodelist), 2): + names.append(nodelist[i][1]) + n = Node('global', names) + n.lineno = nodelist[0][2] + return n + + def exec_stmt(self, nodelist): + # exec_stmt: 'exec' expr ['in' expr [',' expr]] + expr1 = self.com_node(nodelist[1]) + if len(nodelist) >= 4: + expr2 = self.com_node(nodelist[3]) + if len(nodelist) >= 6: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + else: + expr2 = expr3 = None + + n = Node('exec', expr1, expr2, expr3) + n.lineno = nodelist[0][2] + return n + + def assert_stmt(self, nodelist): + # 'assert': test, [',' test] + expr1 = self.com_node(nodelist[1]) + if (len(nodelist) == 4): + expr2 = self.com_node(nodelist[3]) + else: + expr2 = Node('name', 'None') + n = Node('assert', expr1, expr2) + n.lineno = nodelist[0][2] + return n + + def if_stmt(self, nodelist): + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] + tests = [ ] + for i in range(0, len(nodelist) - 3, 4): + testNode = self.com_node(nodelist[i + 1]) + suiteNode = self.com_node(nodelist[i + 3]) + tests.append((testNode, suiteNode)) + + if len(nodelist) % 4 == 3: + elseNode = self.com_node(nodelist[-1]) +## elseNode.lineno = nodelist[-1][1][2] + else: + elseNode = None + n = Node('if', tests, elseNode) + n.lineno = nodelist[0][2] + return n + + def while_stmt(self, nodelist): + # 'while' test ':' suite ['else' ':' suite] + + testNode = self.com_node(nodelist[1]) + bodyNode = self.com_node(nodelist[3]) + + if len(nodelist) > 4: + elseNode = self.com_node(nodelist[6]) + else: + elseNode = None + + n = Node('while', testNode, bodyNode, elseNode) + n.lineno = nodelist[0][2] + return n + + def for_stmt(self, nodelist): + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] + + assignNode = self.com_assign(nodelist[1], OP_ASSIGN) + listNode = self.com_node(nodelist[3]) + bodyNode = self.com_node(nodelist[5]) + + if len(nodelist) > 8: + elseNode = self.com_node(nodelist[8]) + else: + elseNode = None + + n = Node('for', assignNode, listNode, bodyNode, elseNode) + n.lineno = nodelist[0][2] + return n + + def try_stmt(self, nodelist): + # 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] + # | 'try' ':' suite 'finally' ':' suite + if nodelist[3][0] != symbol.except_clause: + return self.com_try_finally(nodelist) + + return self.com_try_except(nodelist) + + def suite(self, nodelist): + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT + if len(nodelist) == 1: + return self.com_stmt(nodelist[0]) + + stmts = [ ] + for node in nodelist: + if node[0] == symbol.stmt: + self.com_append_stmt(stmts, node) + return Node('stmt', stmts) + + # -------------------------------------------------------------- + # + # EXPRESSION NODES (invoked by com_node()) + # + + def testlist(self, nodelist): + # testlist: expr (',' expr)* [','] + # exprlist: expr (',' expr)* [','] + return self.com_binary('tuple', nodelist) + + exprlist = testlist + + def test(self, nodelist): + # and_test ('or' and_test)* | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + return self.com_binary('or', nodelist) + + def and_test(self, nodelist): + # not_test ('and' not_test)* + return self.com_binary('and', nodelist) + + def not_test(self, nodelist): + # 'not' not_test | comparison + result = self.com_node(nodelist[-1]) + if len(nodelist) == 2: + n = Node('not', result) + n.lineno = nodelist[0][2] + return n + return result + + def comparison(self, nodelist): + # comparison: expr (comp_op expr)* + node = self.com_node(nodelist[0]) + if len(nodelist) == 1: + return node + + results = [ ] + for i in range(2, len(nodelist), 2): + nl = nodelist[i-1] + + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' + # | 'in' | 'not' 'in' | 'is' | 'is' 'not' + n = nl[1] + if n[0] == token.NAME: + type = n[1] + if len(nl) == 3: + if type == 'not': + type = 'notin' + else: + type = 'isnot' + else: + type = _cmp_types[n[0]] + + lineno = nl[1][2] + results.append(type, self.com_node(nodelist[i])) + + # we need a special "compare" node so that we can distinguish + # 3 < x < 5 from (3 < x) < 5 + # the two have very different semantics and results (note that the + # latter form is always true) + + n = Node('compare', node, results) + n.lineno = lineno + return n + + def expr(self, nodelist): + # xor_expr ('|' xor_expr)* + return self.com_binary('bitor', nodelist) + + def xor_expr(self, nodelist): + # xor_expr ('^' xor_expr)* + return self.com_binary('bitxor', nodelist) + + def and_expr(self, nodelist): + # xor_expr ('&' xor_expr)* + return self.com_binary('bitand', nodelist) + + def shift_expr(self, nodelist): + # shift_expr ('<<'|'>>' shift_expr)* + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.LEFTSHIFT: + node = Node('<<', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('>>', [node, right]) + node.lineno = nodelist[1][2] + return node + + def arith_expr(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.PLUS: + node = Node('+', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('-', [node, right]) + node.lineno = nodelist[1][2] + return node + + def term(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.STAR: + node = Node('*', [node, right]) + node.lineno = nodelist[1][2] + elif nodelist[i-1][0] == token.SLASH: + node = Node('/', [node, right]) + node.lineno = nodelist[1][2] + else: + node = Node('%', [node, right]) + node.lineno = nodelist[1][2] + return node + + def factor(self, nodelist): + t = nodelist[0][0] + node = self.com_node(nodelist[-1]) + if t == token.PLUS: + node = Node('unary+', node) + node.lineno = nodelist[0][2] + elif t == token.MINUS: + node = Node('unary-', node) + node.lineno = nodelist[0][2] + elif t == token.TILDE: + node = Node('invert', node) + node.lineno = nodelist[0][2] + return node + + def power(self, nodelist): + # power: atom trailer* ('**' factor)* + node = self.com_node(nodelist[0]) + for i in range(1, len(nodelist)): + if nodelist[i][0] == token.DOUBLESTAR: + n = Node('power', [node, self.com_node(nodelist[i+1])]) + n.lineno = nodelist[i][2] + return n + + node = self.com_apply_trailer(node, nodelist[i]) + + return node + + def atom(self, nodelist): + t = nodelist[0][0] + if t == token.LPAR: + if nodelist[1][0] == token.RPAR: + n = Node('const', ()) + n.lineno = nodelist[0][2] + return n + return self.com_node(nodelist[1]) + + if t == token.LSQB: + if nodelist[1][0] == token.RSQB: + n = Node('const', [ ]) + n.lineno = nodelist[0][2] + return n + return self.com_list_constructor(nodelist[1]) + + if t == token.LBRACE: + if nodelist[1][0] == token.RBRACE: + return Node('const', { }) + return self.com_dictmaker(nodelist[1]) + + if t == token.BACKQUOTE: + n = Node('backquote', self.com_node(nodelist[1])) + n.lineno = nodelist[0][2] + return n + + if t == token.NUMBER: + ### need to verify this matches compile.c + k = eval(nodelist[0][1]) + n = Node('const', k) + n.lineno = nodelist[0][2] + return n + + if t == token.STRING: + ### need to verify this matches compile.c + k = '' + for node in nodelist: + k = k + eval(node[1]) + n = Node('const', k) + n.lineno = nodelist[0][2] + return n + + if t == token.NAME: + ### any processing to do? + n = Node('name', nodelist[0][1]) + n.lineno = nodelist[0][2] + return n + + raise error, "unknown node type" + + # -------------------------------------------------------------- + # + # INTERNAL PARSING UTILITIES + # + + def com_node(self, node): + # Note: compile.c has handling in com_node for del_stmt, pass_stmt, + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, + # and compound_stmt. + # We'll just dispatch them. + + # + # A ';' at the end of a line can make a NEWLINE token appear here, + # Render it harmless. (genc discards ('discard', ('const', xxxx)) Nodes) + # + if node[0] == token.NEWLINE: + return Node('discard', Node('const', None)) + + if node[0] not in _legal_node_types: + raise error, 'illegal node passed to com_node: %s' % node[0] + + return self._dispatch[node[0]](node[1:]) + + def com_arglist(self, nodelist): + # varargslist: + # (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME] + # | fpdef ['=' test] (',' fpdef ['=' test])* [','] + # | ('**'|'*' '*') NAME) + # fpdef: NAME | '(' fplist ')' + # fplist: fpdef (',' fpdef)* [','] + names = [ ] + defaults = [ ] + flags = 0 + + i = 0 + while i < len(nodelist): + node = nodelist[i] + if node[0] == token.STAR or node[0] == token.DOUBLESTAR: + if node[0] == token.STAR: + node = nodelist[i+1] + if node[0] == token.NAME: + names.append(node[1]) + flags = flags | CO_VARARGS + i = i + 3 + + if i < len(nodelist): + # should be DOUBLESTAR or STAR STAR + if nodelist[i][0] == token.DOUBLESTAR: + node = nodelist[i+1] + else: + node = nodelist[i+2] + names.append(node[1]) + flags = flags | CO_VARKEYWORDS + + break + + # fpdef: NAME | '(' fplist ')' + names.append(self.com_fpdef(node)) + + i = i + 1 + if i >= len(nodelist): + break + + if nodelist[i][0] == token.EQUAL: + defaults.append(self.com_node(nodelist[i + 1])) + i = i + 2 + elif len(defaults): + # Treat "(a=1, b)" as "(a=1, b=None)" + defaults.append(Node('const', None)) + + i = i + 1 + + return names, defaults, flags + + def com_fpdef(self, node): + # fpdef: NAME | '(' fplist ')' + if node[1][0] == token.LPAR: + return self.com_fplist(node[2]) + return node[1][1] + + def com_fplist(self, node): + # fplist: fpdef (',' fpdef)* [','] + if len(node) == 2: + return self.com_fpdef(node[1]) + list = [ ] + for i in range(1, len(node), 2): + list.append(self.com_fpdef(node[i])) + return tuple(list) + + def com_dotted_name(self, node): + # String together the dotted names and return the string + name = "" + for n in node: + if type(n) == type(()) and n[0] == 1: + name = name + n[1] + '.' + return name[:-1] + + def com_bases(self, node): + bases = [ ] + for i in range(1, len(node), 2): + bases.append(self.com_node(node[i])) + return bases + + def com_try_finally(self, nodelist): + # try_fin_stmt: "try" ":" suite "finally" ":" suite + n = Node('tryfinally', self.com_node(nodelist[2]), self.com_node(nodelist[5])) + n.lineno = nodelist[0][2] + return n + + def com_try_except(self, nodelist): + # try_except: 'try' ':' suite (except_clause ':' suite)* ['else' suite] + #tryexcept: [TryNode, [except_clauses], elseNode)] + stmt = self.com_node(nodelist[2]) + clauses = [] + elseNode = None + for i in range(3, len(nodelist), 3): + node = nodelist[i] + if node[0] == symbol.except_clause: + # except_clause: 'except' [expr [',' expr]] */ + if len(node) > 2: + expr1 = self.com_node(node[2]) + if len(node) > 4: + expr2 = self.com_assign(node[4], OP_ASSIGN) + else: + expr2 = None + else: + expr1 = expr2 = None + clauses.append(expr1, expr2, self.com_node(nodelist[i+2])) + + if node[0] == token.NAME: + elseNode = self.com_node(nodelist[i+2]) + n = Node('tryexcept', self.com_node(nodelist[2]), clauses, elseNode) + n.lineno = nodelist[0][2] + return n + + def com_assign(self, node, assigning): + # return a node suitable for use as an "lvalue" + # loop to avoid trivial recursion + while 1: + t = node[0] + if t == symbol.exprlist or t == symbol.testlist: + if len(node) > 2: + return self.com_assign_tuple(node, assigning) + node = node[1] + elif t in _assign_types: + if len(node) > 2: + raise SyntaxError, "can't assign to operator" + node = node[1] + elif t == symbol.power: + if node[1][0] != symbol.atom: + raise SyntaxError, "can't assign to operator" + if len(node) > 2: + primary = self.com_node(node[1]) + for i in range(2, len(node)-1): + ch = node[i] + if ch[0] == token.DOUBLESTAR: + raise SyntaxError, "can't assign to operator" + primary = self.com_apply_trailer(primary, ch) + return self.com_assign_trailer(primary, node[-1], assigning) + node = node[1] + elif t == symbol.atom: + t = node[1][0] + if t == token.LPAR: + node = node[2] + if node[0] == token.RPAR: + raise SyntaxError, "can't assign to ()" + elif t == token.LSQB: + node = node[2] + if node[0] == token.RSQB: + raise SyntaxError, "can't assign to []" + return self.com_assign_list(node, assigning) + elif t == token.NAME: + return self.com_assign_name(node[1], assigning) + else: + raise SyntaxError, "can't assign to literal" + else: + raise SyntaxError, "bad assignment" + + def com_assign_tuple(self, node, assigning): + assigns = [ ] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return Node('ass_tuple', assigns) + + def com_assign_list(self, node, assigning): + assigns = [ ] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return Node('ass_list', assigns) + + def com_assign_name(self, node, assigning): + return Node('ass_name', node[1], assigning) + + def com_assign_trailer(self, primary, node, assigning): + t = node[1][0] + if t == token.LPAR: + raise SyntaxError, "can't assign to function call" + if t == token.DOT: + return self.com_assign_attr(primary, node[2], assigning) + if t == token.LSQB: + return self.com_subscriptlist(primary, node[2], assigning) + raise SyntaxError, "unknown trailer type: %s" % t + + def com_assign_attr(self, primary, node, assigning): + return Node('ass_attr', primary, node[1], assigning) + + def com_binary(self, type, nodelist): + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." + if len(nodelist) == 1: + return self.com_node(nodelist[0]) + items = [ ] + for i in range(0, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + return Node(type, items) + + def com_stmt(self, node): + #pprint.pprint(node) + result = self.com_node(node) + try: + result[0] + except: + print node[0] + if result[0] == 'stmt': + return result + return Node('stmt', [ result ]) + + def com_append_stmt(self, stmts, node): + result = self.com_node(node) + try: + result[0] + except: + print node + if result[0] == 'stmt': + stmts[len(stmts):] = result[1] + else: + stmts.append(result) + + def com_list_constructor(self, nodelist): + values = [ ] + for i in range(1, len(nodelist), 2): + values.append(self.com_node(nodelist[i])) + return Node('list', values) + + def com_dictmaker(self, nodelist): + # dictmaker: test ':' test (',' test ':' value)* [','] + items = [ ] + for i in range(1, len(nodelist), 4): + items.append(self.com_node(nodelist[i]), self.com_node(nodelist[i+2])) + return Node('dict', items) + + def com_apply_trailer(self, primaryNode, nodelist): + t = nodelist[1][0] + if t == token.LPAR: + return self.com_call_function(primaryNode, nodelist[2]) + if t == token.DOT: + return self.com_select_member(primaryNode, nodelist[2]) + if t == token.LSQB: + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) + + raise SyntaxError, 'unknown node type: %s' % t + + def com_select_member(self, primaryNode, nodelist): + if nodelist[0] != token.NAME: + raise SyntaxError, "member must be a name" + n = Node('getattr', primaryNode, nodelist[1]) + n.lineno = nodelist[2] + return n + + def com_call_function(self, primaryNode, nodelist): + if nodelist[0] == token.RPAR: + return Node('call_func', primaryNode, [ ]) + args = [ ] + kw = 0 + for i in range(1, len(nodelist), 2): + kw, result = self.com_argument(nodelist[i], kw) + args.append(result) + return Node('call_func', primaryNode, args) + + def com_argument(self, nodelist, kw): + if len(nodelist) == 2: + if kw: + raise SyntaxError, "non-keyword arg after keyword arg" + return 0, self.com_node(nodelist[1]) + result = self.com_node(nodelist[3]) + n = nodelist[1] + while len(n) == 2 and n[0] != token.NAME: + n = n[1] + if n[0] != token.NAME: + raise SyntaxError, "keyword can't be an expression (%s)"%n[0] + n = Node('keyword', n[1], result) + n.lineno = result.lineno + return 1, n + + def com_subscriptlist(self, primary, nodelist, assigning): + # slicing: simple_slicing | extended_slicing + # simple_slicing: primary "[" short_slice "]" + # extended_slicing: primary "[" slice_list "]" + # slice_list: slice_item ("," slice_item)* [","] + + # backwards compat slice for '[i:j]' + if len(nodelist) == 2: + sub = nodelist[1] + if (sub[1][0] == token.COLON or \ + (len(sub) > 2 and sub[2][0] == token.COLON)) and \ + sub[-1][0] != symbol.sliceop: + return self.com_slice(primary, sub, assigning) + + subscripts = [ ] + for i in range(1, len(nodelist), 2): + subscripts.append(self.com_subscript(nodelist[i])) + + return Node('subscript', primary, assigning, subscripts) + + def com_subscript(self, node): + # slice_item: expression | proper_slice | ellipsis + ch = node[1] + if ch[0] == token.DOT and node[2][0] == token.DOT: + return ('ellipsis', None) + if ch[0] == token.COLON or len(node) > 2: + return self.com_sliceobj(node) + return self.com_node(ch) + + def com_sliceobj(self, node): + # proper_slice: short_slice | long_slice + # short_slice: [lower_bound] ":" [upper_bound] + # long_slice: short_slice ":" [stride] + # lower_bound: expression + # upper_bound: expression + # stride: expression + # + # Note: a stride may be further slicing... + + items = [ ] + + if node[1][0] == token.COLON: + items.append(Node('const', None)) + i = 2 + else: + items.append(self.com_node(node[1])) + # i == 2 is a COLON + i = 3 + + if i < len(node) and node[i][0] == symbol.test: + items.append(self.com_node(node[i])) + i = i + 1 + else: + items.append(Node('const', None)) + + # a short_slice has been built. look for long_slice now by looking + # for strides... + for j in range(i, len(node)): + ch = node[j] + if len(ch) == 2: + items.append(Node('const', None)) + else: + items.append(self.com_node(ch[2])) + + return Node('sliceobj', items) + + def com_slice(self, primary, node, assigning): + # short_slice: [lower_bound] ":" [upper_bound] + lower = upper = None + if len(node) == 3: + if node[1][0] == token.COLON: + upper = self.com_node(node[2]) + else: + lower = self.com_node(node[1]) + elif len(node) == 4: + lower = self.com_node(node[1]) + upper = self.com_node(node[3]) + return Node('slice', primary, assigning, lower, upper) + + def get_docstring(self, node, n=None): + if n is None: + n = node[0] + node = node[1:] + if n == symbol.suite: + if len(node) == 1: + return self.get_docstring(node[0]) + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.file_input: + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.atom: + if node[0][0] == token.STRING: + s = '' + for t in node: + s = s + eval(t[1]) + return s + return None + if n == symbol.stmt or n == symbol.simple_stmt or n == symbol.small_stmt: + return self.get_docstring(node[0]) + if n in _doc_nodes and len(node) == 1: + return self.get_docstring(node[0]) + return None + + +_doc_nodes = [ + symbol.expr_stmt, + symbol.testlist, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + ] + +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' +# | 'in' | 'not' 'in' | 'is' | 'is' 'not' +_cmp_types = { + token.LESS : '<', + token.GREATER : '>', + token.EQEQUAL : '==', + token.EQUAL : '==', + token.LESSEQUAL : '<=', + token.GREATEREQUAL : '>=', + token.NOTEQUAL : '!=', + } + +_legal_node_types = [ + symbol.funcdef, + symbol.classdef, + symbol.stmt, + symbol.small_stmt, + symbol.flow_stmt, + symbol.simple_stmt, + symbol.compound_stmt, + symbol.expr_stmt, + symbol.print_stmt, + symbol.del_stmt, + symbol.pass_stmt, + symbol.break_stmt, + symbol.continue_stmt, + symbol.return_stmt, + symbol.raise_stmt, + symbol.import_stmt, + symbol.global_stmt, + symbol.exec_stmt, + symbol.assert_stmt, + symbol.if_stmt, + symbol.while_stmt, + symbol.for_stmt, + symbol.try_stmt, + symbol.suite, + symbol.testlist, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.exprlist, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + symbol.atom, + ] + +_assign_types = [ + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + ] + +# Local Variables: +# mode: python +# indent-tabs-mode: nil +# py-indent-offset: 2 +# py-smart-indentation: nil +# End: +