Removed files no longer needed.
This commit is contained in:
Fred Drake
parent
3c1b4a40dc
commit
19405a4a2a
|
|
@ -1,30 +0,0 @@
|
||||||
# Lame substitute for a fine script to generate the table from ast.txt
|
|
||||||
|
|
||||||
from compiler import astgen
|
|
||||||
|
|
||||||
AST_DEF = '../compiler/ast.txt'
|
|
||||||
|
|
||||||
def sort(l):
|
|
||||||
l = l[:]
|
|
||||||
l.sort(lambda a, b: cmp(a.name, b.name))
|
|
||||||
return l
|
|
||||||
|
|
||||||
def main():
|
|
||||||
nodes = astgen.parse_spec(AST_DEF)
|
|
||||||
print "\\begin{longtableiii}{lll}{class}{Node type}{Attribute}{Value}"
|
|
||||||
print
|
|
||||||
for node in sort(nodes):
|
|
||||||
if node.argnames:
|
|
||||||
print "\\lineiii{%s}{%s}{}" % (node.name, node.argnames[0])
|
|
||||||
else:
|
|
||||||
print "\\lineiii{%s}{}{}" % node.name
|
|
||||||
|
|
||||||
for arg in node.argnames[1:]:
|
|
||||||
print "\\lineiii{}{\\member{%s}}{}" % arg
|
|
||||||
print "\\hline", "\n"
|
|
||||||
print "\\end{longtableiii}"
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
||||||
|
|
@ -1,253 +0,0 @@
|
||||||
\begin{longtableiii}{lll}{class}{Node type}{Attribute}{Value}
|
|
||||||
|
|
||||||
\lineiii{Add}{\member{left}}{left operand}
|
|
||||||
\lineiii{}{\member{right}}{right operand}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{And}{\member{nodes}}{list of operands}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{AssAttr}{}{\emph{attribute as target of assignment}}
|
|
||||||
\lineiii{}{\member{expr}}{expression on the left-hand side of the dot}
|
|
||||||
\lineiii{}{\member{attrname}}{the attribute name, a string}
|
|
||||||
\lineiii{}{\member{flags}}{XXX}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{AssList}{\member{nodes}}{list of list elements being assigned to}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{AssName}{\member{name}}{name being assigned to}
|
|
||||||
\lineiii{}{\member{flags}}{XXX}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{AssTuple}{\member{nodes}}{list of tuple elements being assigned to}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Assert}{\member{test}}{the expression to be tested}
|
|
||||||
\lineiii{}{\member{fail}}{the value of the \exception{AssertionError}}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Assign}{\member{nodes}}{a list of assignment targets, one per equal sign}
|
|
||||||
\lineiii{}{\member{expr}}{the value being assigned}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{AugAssign}{\member{node}}{}
|
|
||||||
\lineiii{}{\member{op}}{}
|
|
||||||
\lineiii{}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Backquote}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Bitand}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Bitor}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Bitxor}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Break}{}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{CallFunc}{\member{node}}{expression for the callee}
|
|
||||||
\lineiii{}{\member{args}}{a list of arguments}
|
|
||||||
\lineiii{}{\member{star_args}}{the extended *-arg value}
|
|
||||||
\lineiii{}{\member{dstar_args}}{the extended **-arg value}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Class}{\member{name}}{the name of the class, a string}
|
|
||||||
\lineiii{}{\member{bases}}{a list of base classes}
|
|
||||||
\lineiii{}{\member{doc}}{doc string, a string or \code{None}}
|
|
||||||
\lineiii{}{\member{code}}{the body of the class statement}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Compare}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{ops}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Const}{\member{value}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Continue}{}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Dict}{\member{items}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Discard}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Div}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Ellipsis}{}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Exec}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{locals}}{}
|
|
||||||
\lineiii{}{\member{globals}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{For}{\member{assign}}{}
|
|
||||||
\lineiii{}{\member{list}}{}
|
|
||||||
\lineiii{}{\member{body}}{}
|
|
||||||
\lineiii{}{\member{else_}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{From}{\member{modname}}{}
|
|
||||||
\lineiii{}{\member{names}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Function}{\member{name}}{name used in def, a string}
|
|
||||||
\lineiii{}{\member{argnames}}{list of argument names, as strings}
|
|
||||||
\lineiii{}{\member{defaults}}{list of default values}
|
|
||||||
\lineiii{}{\member{flags}}{xxx}
|
|
||||||
\lineiii{}{\member{doc}}{doc string, a string or \code{None}}
|
|
||||||
\lineiii{}{\member{code}}{the body of the function}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Getattr}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{attrname}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Global}{\member{names}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{If}{\member{tests}}{}
|
|
||||||
\lineiii{}{\member{else_}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Import}{\member{names}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Invert}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Keyword}{\member{name}}{}
|
|
||||||
\lineiii{}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Lambda}{\member{argnames}}{}
|
|
||||||
\lineiii{}{\member{defaults}}{}
|
|
||||||
\lineiii{}{\member{flags}}{}
|
|
||||||
\lineiii{}{\member{code}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{LeftShift}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{List}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{ListComp}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{quals}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{ListCompFor}{\member{assign}}{}
|
|
||||||
\lineiii{}{\member{list}}{}
|
|
||||||
\lineiii{}{\member{ifs}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{ListCompIf}{\member{test}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Mod}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Module}{\member{doc}}{doc string, a string or \code{None}}
|
|
||||||
\lineiii{}{\member{node}}{body of the module, a \class{Stmt}}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Mul}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Name}{\member{name}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Not}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Or}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Pass}{}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Power}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Print}{\member{nodes}}{}
|
|
||||||
\lineiii{}{\member{dest}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Printnl}{\member{nodes}}{}
|
|
||||||
\lineiii{}{\member{dest}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Raise}{\member{expr1}}{}
|
|
||||||
\lineiii{}{\member{expr2}}{}
|
|
||||||
\lineiii{}{\member{expr3}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Return}{\member{value}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{RightShift}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Slice}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{flags}}{}
|
|
||||||
\lineiii{}{\member{lower}}{}
|
|
||||||
\lineiii{}{\member{upper}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Sliceobj}{\member{nodes}}{list of statements}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Stmt}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Sub}{\member{left}}{}
|
|
||||||
\lineiii{}{\member{right}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Subscript}{\member{expr}}{}
|
|
||||||
\lineiii{}{\member{flags}}{}
|
|
||||||
\lineiii{}{\member{subs}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{TryExcept}{\member{body}}{}
|
|
||||||
\lineiii{}{\member{handlers}}{}
|
|
||||||
\lineiii{}{\member{else_}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{TryFinally}{\member{body}}{}
|
|
||||||
\lineiii{}{\member{final}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Tuple}{\member{nodes}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{UnaryAdd}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{UnarySub}{\member{expr}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{While}{\member{test}}{}
|
|
||||||
\lineiii{}{\member{body}}{}
|
|
||||||
\lineiii{}{\member{else_}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\lineiii{Yield}{\member{value}}{}
|
|
||||||
\hline
|
|
||||||
|
|
||||||
\end{longtableiii}
|
|
||||||
|
|
@ -1,380 +0,0 @@
|
||||||
% Complete documentation on the extended LaTeX markup used for Python
|
|
||||||
% documentation is available in ``Documenting Python'', which is part
|
|
||||||
% of the standard documentation for Python. It may be found online
|
|
||||||
% at:
|
|
||||||
%
|
|
||||||
% http://www.python.org/doc/current/doc/doc.html
|
|
||||||
|
|
||||||
\documentclass{howto}
|
|
||||||
|
|
||||||
\title{Python compiler package}
|
|
||||||
|
|
||||||
\author{Jeremy Hylton}
|
|
||||||
|
|
||||||
% Please at least include a long-lived email address;
|
|
||||||
% the rest is at your discretion.
|
|
||||||
\authoraddress{
|
|
||||||
PythonLabs \\
|
|
||||||
Zope Corporation \\
|
|
||||||
Email: \email{jeremy@zope.com}
|
|
||||||
}
|
|
||||||
|
|
||||||
\date{August 15, 2001} % update before release!
|
|
||||||
% Use an explicit date so that reformatting
|
|
||||||
% doesn't cause a new date to be used. Setting
|
|
||||||
% the date to \today can be used during draft
|
|
||||||
% stages to make it easier to handle versions.
|
|
||||||
|
|
||||||
\release{2.2} % release version; this is used to define the
|
|
||||||
% \version macro
|
|
||||||
|
|
||||||
\makeindex % tell \index to actually write the .idx file
|
|
||||||
\makemodindex % If this contains a lot of module sections.
|
|
||||||
|
|
||||||
|
|
||||||
\begin{document}
|
|
||||||
|
|
||||||
\maketitle
|
|
||||||
|
|
||||||
\begin{abstract}
|
|
||||||
|
|
||||||
\noindent
|
|
||||||
The Python compiler package is a tool for analyzing Python source code
|
|
||||||
and generating Python bytecode. The compiler contains libraries to
|
|
||||||
generate an abstract syntax tree from Python source code and to
|
|
||||||
generate Python bytecode from the tree.
|
|
||||||
|
|
||||||
\end{abstract}
|
|
||||||
|
|
||||||
\tableofcontents
|
|
||||||
|
|
||||||
|
|
||||||
\section{Introduction\label{Introduction}}
|
|
||||||
|
|
||||||
The \module{compiler} package is a Python source to bytecode
|
|
||||||
translator written in Python. It uses the builtin parser and standard
|
|
||||||
\ulink{\module{parser}}
|
|
||||||
{http://www.python.org/doc/current/lib/module-parser.html} to
|
|
||||||
generated a concrete syntax tree. This tree is used to generate an
|
|
||||||
abstract syntax tree (AST) and then Python bytecode.
|
|
||||||
|
|
||||||
The full functionality of the package duplicates the builtin compiler
|
|
||||||
provided with the Python interpreter. It is intended to match its
|
|
||||||
behavior almost exactly. Why implement another compiler that does the
|
|
||||||
same thing? The package is useful for a variety of purposes. It can
|
|
||||||
be modified more easily than the builtin compiler. The AST it
|
|
||||||
generates is useful for analyzing Python source code.
|
|
||||||
|
|
||||||
This manual explains how the various components of the
|
|
||||||
\module{compiler} package work. It blends reference material with a
|
|
||||||
tutorial. (At least it will when the document is done.)
|
|
||||||
|
|
||||||
\subsection{The basic interface}
|
|
||||||
|
|
||||||
\declaremodule{}{compiler}
|
|
||||||
|
|
||||||
The top-level of the package defines four functions. If you import
|
|
||||||
\module{compiler}, you will get these functions and a collection of
|
|
||||||
modules contained in the package.
|
|
||||||
|
|
||||||
\begin{funcdesc}{parse}{buf}
|
|
||||||
Returns an abstract syntax tree for the Python source code in \var{buf}.
|
|
||||||
The function raises SyntaxError if there is an error in the source
|
|
||||||
code. The return value is a \class{compiler.ast.Module} instance that
|
|
||||||
contains the tree.
|
|
||||||
\end{funcdesc}
|
|
||||||
|
|
||||||
\begin{funcdesc}{parseFile}{path}
|
|
||||||
Return an abstract syntax tree for the Python source code in the file
|
|
||||||
specified by \var{path}. It is equivalent to
|
|
||||||
\code{parse(open(\var{path}).read())}.
|
|
||||||
\end{funcdesc}
|
|
||||||
|
|
||||||
\begin{funcdesc}{walk}{ast, visitor\optional{, verbose}}
|
|
||||||
Do a pre-order walk over the abstract syntax tree \var{ast}. Call the
|
|
||||||
appropriate method on the \var{visitor} instance for each node
|
|
||||||
encountered.
|
|
||||||
\end{funcdesc}
|
|
||||||
|
|
||||||
\begin{funcdesc}{compile}{path}
|
|
||||||
Compile the file \var{path} and generate the corresponding \file{.pyc}
|
|
||||||
file.
|
|
||||||
\end{funcdesc}
|
|
||||||
|
|
||||||
The \module{compiler} package contains the following modules:
|
|
||||||
\refmodule[compiler.ast]{ast}, \module{consts}, \module{future},
|
|
||||||
\module{misc}, \module{pyassem}, \module{pycodegen}, \module{symbols},
|
|
||||||
\module{transformer}, and \refmodule[compiler.visitor]{visitor}.
|
|
||||||
|
|
||||||
\subsection{Limitations}
|
|
||||||
|
|
||||||
There are some problems with the error checking of the compiler
|
|
||||||
package. The interpreter detects syntax errors in two distinct
|
|
||||||
phases. One set of errors is detected by the interpreter's parser,
|
|
||||||
the other set by the compiler. The compiler package relies on the
|
|
||||||
interpreter's parser, so it get the first phases of error checking for
|
|
||||||
free. It implements the second phase itself, and that implement is
|
|
||||||
incomplete. For example, the compiler package does not raise an error
|
|
||||||
if a name appears more than once in an argument list:
|
|
||||||
\code{def f(x, x): ...}
|
|
||||||
|
|
||||||
A future version of the compiler should fix these problems.
|
|
||||||
|
|
||||||
\section{Python Abstract Syntax}
|
|
||||||
|
|
||||||
The \module{compiler.ast} module defines an abstract syntax for
|
|
||||||
Python. In the abstract syntax tree, each node represents a syntactic
|
|
||||||
construct. The root of the tree is \class{Module} object.
|
|
||||||
|
|
||||||
The abstract syntax offers a higher level interface to parsed Python
|
|
||||||
source code. The \ulink{\module{parser}}
|
|
||||||
{http://www.python.org/doc/current/lib/module-parser.html}
|
|
||||||
module and the compiler written in C for the Python interpreter use a
|
|
||||||
concrete syntax tree. The concrete syntax is tied closely to the
|
|
||||||
grammar description used for the Python parser. Instead of a single
|
|
||||||
node for a construct, there are often several levels of nested nodes
|
|
||||||
that are introduced by Python's precedence rules.
|
|
||||||
|
|
||||||
The abstract syntax tree is created by the
|
|
||||||
\module{compiler.transformer} module. The transformer relies on the
|
|
||||||
builtin Python parser to generate a concrete syntax tree. It
|
|
||||||
generates an abstract syntax tree from the concrete tree.
|
|
||||||
|
|
||||||
The \module{transformer} module was created by Greg
|
|
||||||
Stein\index{Stein, Greg} and Bill Tutt\index{Tutt, Bill} for an
|
|
||||||
experimental Python-to-C compiler. The current version contains a
|
|
||||||
number of modifications and improvements, but the basic form of the
|
|
||||||
abstract syntax and of the transformer are due to Stein and Tutt.
|
|
||||||
|
|
||||||
\subsection{AST Nodes}
|
|
||||||
|
|
||||||
\declaremodule{}{compiler.ast}
|
|
||||||
|
|
||||||
The \module{compiler.ast} module is generated from a text file that
|
|
||||||
describes each node type and its elements. Each node type is
|
|
||||||
represented as a class that inherits from the abstract base class
|
|
||||||
\class{compiler.ast.Node} and defines a set of named attributes for
|
|
||||||
child nodes.
|
|
||||||
|
|
||||||
\begin{classdesc}{Node}{}
|
|
||||||
|
|
||||||
The \class{Node} instances are created automatically by the parser
|
|
||||||
generator. The recommended interface for specific \class{Node}
|
|
||||||
instances is to use the public attributes to access child nodes. A
|
|
||||||
public attribute may be bound to a single node or to a sequence of
|
|
||||||
nodes, depending on the \class{Node} type. For example, the
|
|
||||||
\member{bases} attribute of the \class{Class} node, is bound to a
|
|
||||||
list of base class nodes, and the \member{doc} attribute is bound to
|
|
||||||
a single node.
|
|
||||||
|
|
||||||
Each \class{Node} instance has a \member{lineno} attribute which may
|
|
||||||
be \code{None}. XXX Not sure what the rules are for which nodes
|
|
||||||
will have a useful lineno.
|
|
||||||
\end{classdesc}
|
|
||||||
|
|
||||||
All \class{Node} objects offer the following methods:
|
|
||||||
|
|
||||||
\begin{methoddesc}{getChildren}{}
|
|
||||||
Returns a flattened list of the child nodes and objects in the
|
|
||||||
order they occur. Specifically, the order of the nodes is the
|
|
||||||
order in which they appear in the Python grammar. Not all of the
|
|
||||||
children are \class{Node} instances. The names of functions and
|
|
||||||
classes, for example, are plain strings.
|
|
||||||
\end{methoddesc}
|
|
||||||
|
|
||||||
\begin{methoddesc}{getChildNodes}{}
|
|
||||||
Returns a flattened list of the child nodes in the order they
|
|
||||||
occur. This method is like \method{getChildren()}, except that it
|
|
||||||
only returns those children that are \class{Node} instances.
|
|
||||||
\end{methoddesc}
|
|
||||||
|
|
||||||
Two examples illustrate the general structure of \class{Node}
|
|
||||||
classes. The \keyword{while} statement is defined by the following
|
|
||||||
grammar production:
|
|
||||||
|
|
||||||
\begin{verbatim}
|
|
||||||
while_stmt: "while" expression ":" suite
|
|
||||||
["else" ":" suite]
|
|
||||||
\end{verbatim}
|
|
||||||
|
|
||||||
The \class{While} node has three attributes: \member{test},
|
|
||||||
\member{body}, and \member{else_}. (If the natural name for an
|
|
||||||
attribute is also a Python reserved word, it can't be used as an
|
|
||||||
attribute name. An underscore is appended to the word to make it a
|
|
||||||
legal identifier, hence \member{else_} instead of \keyword{else}.)
|
|
||||||
|
|
||||||
The \keyword{if} statement is more complicated because it can include
|
|
||||||
several tests.
|
|
||||||
|
|
||||||
\begin{verbatim}
|
|
||||||
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
|
||||||
\end{verbatim}
|
|
||||||
|
|
||||||
The \class{If} node only defines two attributes: \member{tests} and
|
|
||||||
\member{else_}. The \member{tests} attribute is a sequence of test
|
|
||||||
expression, consequent body pairs. There is one pair for each
|
|
||||||
\keyword{if}/\keyword{elif} clause. The first element of the pair is
|
|
||||||
the test expression. The second elements is a \class{Stmt} node that
|
|
||||||
contains the code to execute if the test is true.
|
|
||||||
|
|
||||||
The \method{getChildren()} method of \class{If} returns a flat list of
|
|
||||||
child nodes. If there are three \keyword{if}/\keyword{elif} clauses
|
|
||||||
and no \keyword{else} clause, then \method{getChildren()} will return
|
|
||||||
a list of six elements: the first test expression, the first
|
|
||||||
\class{Stmt}, the second text expression, etc.
|
|
||||||
|
|
||||||
The following table lists each of the \class{Node} subclasses defined
|
|
||||||
in \module{compiler.ast} and each of the public attributes available
|
|
||||||
on their instances. The values of most of the attributes are
|
|
||||||
themselves \class{Node} instances or sequences of instances. When the
|
|
||||||
value is something other than an instance, the type is noted in the
|
|
||||||
comment. The attributes are listed in the order in which they are
|
|
||||||
returned by \method{getChildren()} and \method{getChildNodes()}.
|
|
||||||
|
|
||||||
\input{asttable}
|
|
||||||
|
|
||||||
|
|
||||||
\subsection{Assignment nodes}
|
|
||||||
|
|
||||||
There is a collection of nodes used to represent assignments. Each
|
|
||||||
assignment statement in the source code becomes a single
|
|
||||||
\class{Assign} node in the AST. The \member{nodes} attribute is a
|
|
||||||
list that contains a node for each assignment target. This is
|
|
||||||
necessary because assignment can be chained, e.g. \code{a = b = 2}.
|
|
||||||
Each \class{Node} in the list will be one of the following classes:
|
|
||||||
\class{AssAttr}, \class{AssList}, \class{AssName}, or
|
|
||||||
\class{AssTuple}.
|
|
||||||
|
|
||||||
Each target assignment node will describe the kind of object being
|
|
||||||
assigned to: \class{AssName} for a simple name, e.g. \code{a = 1}.
|
|
||||||
\class{AssAttr} for an attribute assigned, e.g. \code{a.x = 1}.
|
|
||||||
\class{AssList} and \class{AssTuple} for list and tuple expansion
|
|
||||||
respectively, e.g. \code{a, b, c = a_tuple}.
|
|
||||||
|
|
||||||
The target assignment nodes also have a \member{flags} attribute that
|
|
||||||
indicates whether the node is being used for assignment or in a delete
|
|
||||||
statement. The \class{AssName} is also used to represent a delete
|
|
||||||
statement, e.g. \class{del x}.
|
|
||||||
|
|
||||||
When an expression contains several attribute references, an
|
|
||||||
assignment or delete statement will contain only one \class{AssAttr}
|
|
||||||
node -- for the final attribute reference. The other attribute
|
|
||||||
references will be represented as \class{Getattr} nodes in the
|
|
||||||
\member{expr} attribute of the \class{AssAttr} instance.
|
|
||||||
|
|
||||||
\subsection{Examples}
|
|
||||||
|
|
||||||
This section shows several simple examples of ASTs for Python source
|
|
||||||
code. The examples demonstrate how to use the \function{parse()}
|
|
||||||
function, what the repr of an AST looks like, and how to access
|
|
||||||
attributes of an AST node.
|
|
||||||
|
|
||||||
The first module defines a single function. Assume it is stored in
|
|
||||||
\file{/tmp/doublelib.py}.
|
|
||||||
|
|
||||||
\begin{verbatim}
|
|
||||||
"""This is an example module.
|
|
||||||
|
|
||||||
This is the docstring.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def double(x):
|
|
||||||
"Return twice the argument"
|
|
||||||
return x * 2
|
|
||||||
\end{verbatim}
|
|
||||||
|
|
||||||
In the interactive interpreter session below, I have reformatted the
|
|
||||||
long AST reprs for readability. The AST reprs use unqualified class
|
|
||||||
names. If you want to create an instance from a repr, you must import
|
|
||||||
the class names from the \module{compiler.ast} module.
|
|
||||||
|
|
||||||
\begin{verbatim}
|
|
||||||
>>> import compiler
|
|
||||||
>>> mod = compiler.parseFile("/tmp/doublelib.py")
|
|
||||||
>>> mod
|
|
||||||
Module('This is an example module.\n\nThis is the docstring.\n',
|
|
||||||
Stmt([Function('double', ['x'], [], 0, 'Return twice the argument',
|
|
||||||
Stmt([Return(Mul((Name('x'), Const(2))))]))]))
|
|
||||||
>>> from compiler.ast import *
|
|
||||||
>>> Module('This is an example module.\n\nThis is the docstring.\n',
|
|
||||||
... Stmt([Function('double', ['x'], [], 0, 'Return twice the argument',
|
|
||||||
... Stmt([Return(Mul((Name('x'), Const(2))))]))]))
|
|
||||||
Module('This is an example module.\n\nThis is the docstring.\n',
|
|
||||||
Stmt([Function('double', ['x'], [], 0, 'Return twice the argument',
|
|
||||||
Stmt([Return(Mul((Name('x'), Const(2))))]))]))
|
|
||||||
>>> mod.doc
|
|
||||||
'This is an example module.\n\nThis is the docstring.\n'
|
|
||||||
>>> for node in mod.node.nodes:
|
|
||||||
... print node
|
|
||||||
...
|
|
||||||
Function('double', ['x'], [], 0, 'Return twice the argument',
|
|
||||||
Stmt([Return(Mul((Name('x'), Const(2))))]))
|
|
||||||
>>> func = mod.node.nodes[0]
|
|
||||||
>>> func.code
|
|
||||||
Stmt([Return(Mul((Name('x'), Const(2))))])
|
|
||||||
\end{verbatim}
|
|
||||||
|
|
||||||
\section{Using Visitors to Walk ASTs}
|
|
||||||
|
|
||||||
\declaremodule{}{compiler.visitor}
|
|
||||||
|
|
||||||
The visitor pattern is ... The \refmodule{compiler} package uses a
|
|
||||||
variant on the visitor pattern that takes advantage of Python's
|
|
||||||
introspection features to elminiate the need for much of the visitor's
|
|
||||||
infrastructure.
|
|
||||||
|
|
||||||
The classes being visited do not need to be programmed to accept
|
|
||||||
visitors. The visitor need only define visit methods for classes it
|
|
||||||
is specifically interested in; a default visit method can handle the
|
|
||||||
rest.
|
|
||||||
|
|
||||||
XXX The magic \method{visit()} method for visitors.
|
|
||||||
|
|
||||||
\begin{funcdesc}{walk}{tree, visitor\optional{, verbose}}
|
|
||||||
\end{funcdesc}
|
|
||||||
|
|
||||||
\begin{classdesc}{ASTVisitor}{}
|
|
||||||
|
|
||||||
The \class{ASTVisitor} is responsible for walking over the tree in the
|
|
||||||
correct order. A walk begins with a call to \method{preorder()}. For
|
|
||||||
each node, it checks the \var{visitor} argument to \method{preorder()}
|
|
||||||
for a method named `visitNodeType,' where NodeType is the name of the
|
|
||||||
node's class, e.g. for a \class{While} node a \method{visitWhile()}
|
|
||||||
would be called. If the method exists, it is called with the node as
|
|
||||||
its first argument.
|
|
||||||
|
|
||||||
The visitor method for a particular node type can control how child
|
|
||||||
nodes are visited during the walk. The \class{ASTVisitor} modifies
|
|
||||||
the visitor argument by adding a visit method to the visitor; this
|
|
||||||
method can be used to visit a particular child node. If no visitor is
|
|
||||||
found for a particular node type, the \method{default()} method is
|
|
||||||
called.
|
|
||||||
\end{classdesc}
|
|
||||||
|
|
||||||
\class{ASTVisitor} objects have the following methods:
|
|
||||||
|
|
||||||
XXX describe extra arguments
|
|
||||||
|
|
||||||
\begin{methoddesc}{default}{node\optional{, \moreargs}}
|
|
||||||
\end{methoddesc}
|
|
||||||
|
|
||||||
\begin{methoddesc}{dispatch}{node\optional{, \moreargs}}
|
|
||||||
\end{methoddesc}
|
|
||||||
|
|
||||||
\begin{methoddesc}{preorder}{tree, visitor}
|
|
||||||
\end{methoddesc}
|
|
||||||
|
|
||||||
|
|
||||||
\section{Bytecode Generation}
|
|
||||||
|
|
||||||
The code generator is a visitor that emits bytecodes. Each visit method
|
|
||||||
can call the \method{emit()} method to emit a new bytecode. The basic
|
|
||||||
code generator is specialized for modules, classes, and functions. An
|
|
||||||
assembler converts that emitted instructions to the low-level bytecode
|
|
||||||
format. It handles things like generator of constant lists of code
|
|
||||||
objects and calculation of jump offsets.
|
|
||||||
|
|
||||||
|
|
||||||
\input{compiler.ind} % Index
|
|
||||||
|
|
||||||
\end{document}
|
|
||||||
Loading…
Reference in New Issue