SF patch [ 597919 ] compiler package and SET_LINENO
A variety of changes from Michael Hudson to get the compiler working with 2.3. The primary change is the handling of SET_LINENO: # The set_lineno() function and the explicit emit() calls for # SET_LINENO below are only used to generate the line number table. # As of Python 2.3, the interpreter does not have a SET_LINENO # instruction. pyassem treats SET_LINENO opcodes as a special case. A few other small changes: - Remove unused code from pycodegen and pyassem. - Fix error handling in parsermodule. When PyParser_SimplerParseString() fails, it sets an exception with detailed info. The parsermodule was clobbering that exception and replacing it was a generic "could not parse string" exception. Keep the original exception.
This commit is contained in:
Jeremy Hylton
parent
436eadd455
commit
accb62b28e
|
|
@ -6,15 +6,8 @@ import sys
|
|||
import types
|
||||
|
||||
from compiler import misc
|
||||
from compiler.consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, \
|
||||
CO_VARKEYWORDS
|
||||
|
||||
def xxx_sort(l):
|
||||
l = l[:]
|
||||
def sorter(a, b):
|
||||
return cmp(a.bid, b.bid)
|
||||
l.sort(sorter)
|
||||
return l
|
||||
from compiler.consts \
|
||||
import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
|
||||
|
||||
class FlowGraph:
|
||||
def __init__(self):
|
||||
|
|
@ -77,7 +70,7 @@ class FlowGraph:
|
|||
def emit(self, *inst):
|
||||
if self._debug:
|
||||
print "\t", inst
|
||||
if inst[0] == 'RETURN_VALUE':
|
||||
if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']:
|
||||
self.current.addOutEdge(self.exit)
|
||||
if len(inst) == 2 and isinstance(inst[1], Block):
|
||||
self.current.addOutEdge(inst[1])
|
||||
|
|
@ -266,7 +259,7 @@ class Block:
|
|||
self.next.append(block)
|
||||
assert len(self.next) == 1, map(str, self.next)
|
||||
|
||||
_uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
|
||||
_uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE',
|
||||
'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
|
||||
|
||||
def pruneNext(self):
|
||||
|
|
@ -443,7 +436,7 @@ class PyFlowGraph(FlowGraph):
|
|||
insts.append(inst)
|
||||
if len(inst) == 1:
|
||||
pc = pc + 1
|
||||
else:
|
||||
elif inst[0] != "SET_LINENO":
|
||||
# arg takes 2 bytes
|
||||
pc = pc + 3
|
||||
end[b] = pc
|
||||
|
|
@ -452,7 +445,7 @@ class PyFlowGraph(FlowGraph):
|
|||
inst = insts[i]
|
||||
if len(inst) == 1:
|
||||
pc = pc + 1
|
||||
else:
|
||||
elif inst[0] != "SET_LINENO":
|
||||
pc = pc + 3
|
||||
opname = inst[0]
|
||||
if self.hasjrel.has_elt(opname):
|
||||
|
|
@ -580,6 +573,7 @@ class PyFlowGraph(FlowGraph):
|
|||
oparg = t[1]
|
||||
if opname == "SET_LINENO":
|
||||
lnotab.nextLine(oparg)
|
||||
continue
|
||||
hi, lo = twobyte(oparg)
|
||||
try:
|
||||
lnotab.addCode(self.opnum[opname], lo, hi)
|
||||
|
|
@ -697,7 +691,7 @@ class LineAddrTable:
|
|||
# after the loading of "b". This works with the C Python
|
||||
# compiler because it only generates a SET_LINENO instruction
|
||||
# for the assignment.
|
||||
if line > 0:
|
||||
if line >= 0:
|
||||
push = self.lnotab.append
|
||||
while addr > 255:
|
||||
push(255); push(0)
|
||||
|
|
@ -768,6 +762,7 @@ class StackDepthTracker:
|
|||
# PRINT_EXPR?
|
||||
'PRINT_ITEM': -1,
|
||||
'RETURN_VALUE': -1,
|
||||
'YIELD_VALUE': -1,
|
||||
'EXEC_STMT': -3,
|
||||
'BUILD_CLASS': -2,
|
||||
'STORE_NAME': -1,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
|
|||
CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, CO_FUTURE_DIVISION
|
||||
from compiler.pyassem import TupleArg
|
||||
|
||||
# XXX The version-specific code can go, since this code only works with 2.x.
|
||||
# Do we have Python 1.x or Python 2.x?
|
||||
try:
|
||||
VERSION = sys.version_info[0]
|
||||
|
|
@ -32,22 +33,14 @@ EXCEPT = 2
|
|||
TRY_FINALLY = 3
|
||||
END_FINALLY = 4
|
||||
|
||||
# XXX this doesn't seem to be used
|
||||
class BlockStack(misc.Stack):
|
||||
__super_init = misc.Stack.__init__
|
||||
|
||||
def __init__(self):
|
||||
self.__super_init(self)
|
||||
self.loop = None
|
||||
|
||||
def compileFile(filename, display=0):
|
||||
f = open(filename)
|
||||
f = open(filename, 'U')
|
||||
buf = f.read()
|
||||
f.close()
|
||||
mod = Module(buf, filename)
|
||||
try:
|
||||
mod.compile(display)
|
||||
except SyntaxError, err:
|
||||
except SyntaxError:
|
||||
raise
|
||||
else:
|
||||
f = open(filename + "c", "wb")
|
||||
|
|
@ -134,7 +127,7 @@ class Module(AbstractCompileMode):
|
|||
# to indicate the type of the value. simplest way to get the
|
||||
# same effect is to call marshal and then skip the code.
|
||||
mtime = os.path.getmtime(self.filename)
|
||||
mtime = struct.pack('i', mtime)
|
||||
mtime = struct.pack('<i', mtime)
|
||||
return self.MAGIC + mtime
|
||||
|
||||
class LocalNameFinder:
|
||||
|
|
@ -310,9 +303,17 @@ class CodeGenerator:
|
|||
else:
|
||||
self.emit(prefix + '_NAME', name)
|
||||
|
||||
def set_lineno(self, node, force=0):
|
||||
"""Emit SET_LINENO if node has lineno attribute and it is
|
||||
different than the last lineno emitted.
|
||||
# The set_lineno() function and the explicit emit() calls for
|
||||
# SET_LINENO below are only used to generate the line number table.
|
||||
# As of Python 2.3, the interpreter does not have a SET_LINENO
|
||||
# instruction. pyassem treats SET_LINENO opcodes as a special case.
|
||||
|
||||
def set_lineno(self, node, force=False):
|
||||
"""Emit SET_LINENO if necessary.
|
||||
|
||||
The instruction is considered necessary if the node has a
|
||||
lineno attribute and it is different than the last lineno
|
||||
emitted.
|
||||
|
||||
Returns true if SET_LINENO was emitted.
|
||||
|
||||
|
|
@ -326,8 +327,8 @@ class CodeGenerator:
|
|||
or force):
|
||||
self.emit('SET_LINENO', lineno)
|
||||
self.last_lineno = lineno
|
||||
return 1
|
||||
return 0
|
||||
return True
|
||||
return False
|
||||
|
||||
# The first few visitor methods handle nodes that generator new
|
||||
# code objects. They use class attributes to determine what
|
||||
|
|
@ -387,9 +388,6 @@ class CodeGenerator:
|
|||
def visitClass(self, node):
|
||||
gen = self.ClassGen(node, self.scopes,
|
||||
self.get_module())
|
||||
if node.doc:
|
||||
self.emit('LOAD_CONST', node.doc)
|
||||
self.storeName('__doc__')
|
||||
walk(node.code, gen)
|
||||
gen.finish()
|
||||
self.set_lineno(node)
|
||||
|
|
@ -447,7 +445,7 @@ class CodeGenerator:
|
|||
self.nextBlock(loop)
|
||||
self.setups.push((LOOP, loop))
|
||||
|
||||
self.set_lineno(node, force=1)
|
||||
self.set_lineno(node, force=True)
|
||||
self.visit(node.test)
|
||||
self.emit('JUMP_IF_FALSE', else_ or after)
|
||||
|
||||
|
|
@ -617,7 +615,7 @@ class CodeGenerator:
|
|||
return start, anchor
|
||||
|
||||
def visitListCompIf(self, node, branch):
|
||||
self.set_lineno(node, force=1)
|
||||
self.set_lineno(node, force=True)
|
||||
self.visit(node.test)
|
||||
self.emit('JUMP_IF_FALSE', branch)
|
||||
self.newBlock()
|
||||
|
|
@ -975,7 +973,7 @@ class CodeGenerator:
|
|||
def visitYield(self, node):
|
||||
self.set_lineno(node)
|
||||
self.visit(node.value)
|
||||
self.emit('YIELD_STMT')
|
||||
self.emit('YIELD_VALUE')
|
||||
|
||||
# slice and subscript stuff
|
||||
|
||||
|
|
@ -1266,7 +1264,6 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
|
|||
self.__super_init(func, scopes, isLambda, class_name, mod)
|
||||
self.graph.setFreeVars(self.scope.get_free_vars())
|
||||
self.graph.setCellVars(self.scope.get_cell_vars())
|
||||
if self.graph.checkFlag(CO_GENERATOR_ALLOWED):
|
||||
if self.scope.generator is not None:
|
||||
self.graph.setFlag(CO_GENERATOR)
|
||||
|
||||
|
|
@ -1304,6 +1301,12 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
|
|||
self.__super_init(klass, scopes, module)
|
||||
self.graph.setFreeVars(self.scope.get_free_vars())
|
||||
self.graph.setCellVars(self.scope.get_cell_vars())
|
||||
self.set_lineno(klass)
|
||||
self.emit("LOAD_GLOBAL", "__name__")
|
||||
self.storeName("__module__")
|
||||
if klass.doc:
|
||||
self.emit("LOAD_CONST", klass.doc)
|
||||
self.storeName('__doc__')
|
||||
|
||||
def generateArgList(arglist):
|
||||
"""Generate an arg list marking TupleArgs"""
|
||||
|
|
@ -1379,7 +1382,5 @@ def wrap_aug(node):
|
|||
return wrapper[node.__class__](node)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
for file in sys.argv[1:]:
|
||||
compileFile(file)
|
||||
|
|
|
|||
|
|
@ -249,6 +249,9 @@ class SymbolVisitor:
|
|||
scope = ClassScope(node.name, self.module)
|
||||
if parent.nested or isinstance(parent, FunctionScope):
|
||||
scope.nested = 1
|
||||
if node.doc is not None:
|
||||
scope.add_def('__doc__')
|
||||
scope.add_def('__module__')
|
||||
self.scopes[node] = scope
|
||||
prev = self.klass
|
||||
self.klass = node.name
|
||||
|
|
|
|||
|
|
@ -37,7 +37,11 @@ from consts import OP_ASSIGN, OP_DELETE, OP_APPLY
|
|||
|
||||
def parseFile(path):
|
||||
f = open(path)
|
||||
src = f.read()
|
||||
# XXX The parser API tolerates files without a trailing newline,
|
||||
# but not strings without a trailing newline. Always add an extra
|
||||
# newline to the file contents, since we're going through the string
|
||||
# version of the API.
|
||||
src = f.read() + "\n"
|
||||
f.close()
|
||||
return parse(src)
|
||||
|
||||
|
|
@ -100,6 +104,7 @@ class Transformer:
|
|||
token.STRING: self.atom_string,
|
||||
token.NAME: self.atom_name,
|
||||
}
|
||||
self.encoding = None
|
||||
|
||||
def transform(self, tree):
|
||||
"""Transform an AST into a modified parse tree."""
|
||||
|
|
@ -110,6 +115,7 @@ class Transformer:
|
|||
def parsesuite(self, text):
|
||||
"""Return a modified parse tree for the given suite text."""
|
||||
# Hack for handling non-native line endings on non-DOS like OSs.
|
||||
# this can go now we have universal newlines?
|
||||
text = text.replace('\x0d', '')
|
||||
return self.transform(parser.suite(text))
|
||||
|
||||
|
|
@ -131,6 +137,12 @@ class Transformer:
|
|||
def compile_node(self, node):
|
||||
### emit a line-number node?
|
||||
n = node[0]
|
||||
|
||||
if n == symbol.encoding_decl:
|
||||
self.encoding = node[2]
|
||||
node = node[1]
|
||||
n = node[0]
|
||||
|
||||
if n == symbol.single_input:
|
||||
return self.single_input(node[1:])
|
||||
if n == symbol.file_input:
|
||||
|
|
@ -519,6 +531,7 @@ class Transformer:
|
|||
return self.com_binary(Tuple, nodelist)
|
||||
|
||||
testlist_safe = testlist # XXX
|
||||
testlist1 = testlist
|
||||
exprlist = testlist
|
||||
|
||||
def test(self, nodelist):
|
||||
|
|
@ -637,11 +650,14 @@ class Transformer:
|
|||
def factor(self, nodelist):
|
||||
elt = nodelist[0]
|
||||
t = elt[0]
|
||||
print "source", nodelist[-1]
|
||||
node = self.com_node(nodelist[-1])
|
||||
# need to handle (unary op)constant here...
|
||||
if t == token.PLUS:
|
||||
node = UnaryAdd(node)
|
||||
node.lineno = elt[2]
|
||||
elif t == token.MINUS:
|
||||
print node
|
||||
node = UnarySub(node)
|
||||
node.lineno = elt[2]
|
||||
elif t == token.TILDE:
|
||||
|
|
@ -699,11 +715,21 @@ class Transformer:
|
|||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
|
||||
def decode_literal(self, lit):
|
||||
if self.encoding:
|
||||
# this is particularly fragile & a bit of a
|
||||
# hack... changes in compile.c:parsestr and
|
||||
# tokenizer.c must be reflected here.
|
||||
if self.encoding not in ['utf-8', 'iso-8859-1']:
|
||||
lit = unicode(lit, 'utf-8').encode(self.encoding)
|
||||
return eval("# coding: %s\n%s" % (self.encoding, lit))
|
||||
else:
|
||||
return eval(lit)
|
||||
|
||||
def atom_string(self, nodelist):
|
||||
### need to verify this matches compile.c
|
||||
k = ''
|
||||
for node in nodelist:
|
||||
k = k + eval(node[1])
|
||||
k += self.decode_literal(node[1])
|
||||
n = Const(k)
|
||||
n.lineno = nodelist[0][2]
|
||||
return n
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ node2tuple(node *n, /* node to convert */
|
|||
PyObject *v;
|
||||
PyObject *w;
|
||||
|
||||
v = mkseq(1 + NCH(n));
|
||||
v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
|
||||
if (v == NULL)
|
||||
return (v);
|
||||
w = PyInt_FromLong(TYPE(n));
|
||||
|
|
@ -106,6 +106,9 @@ node2tuple(node *n, /* node to convert */
|
|||
}
|
||||
(void) addelem(v, i+1, w);
|
||||
}
|
||||
|
||||
if (TYPE(n) == encoding_decl)
|
||||
(void) addelem(v, i+1, PyString_FromString(STR(n)));
|
||||
return (v);
|
||||
}
|
||||
else if (ISTERMINAL(TYPE(n))) {
|
||||
|
|
@ -478,7 +481,7 @@ err_string(char *message)
|
|||
/* PyObject* parser_do_parse(PyObject* args, int type)
|
||||
*
|
||||
* Internal function to actually execute the parse and return the result if
|
||||
* successful, or set an exception if not.
|
||||
* successful or set an exception if not.
|
||||
*
|
||||
*/
|
||||
static PyObject*
|
||||
|
|
@ -494,10 +497,8 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
|
|||
(type == PyST_EXPR)
|
||||
? eval_input : file_input);
|
||||
|
||||
if (n != 0)
|
||||
if (n)
|
||||
res = parser_newstobject(n, type);
|
||||
else
|
||||
err_string("could not parse string");
|
||||
}
|
||||
return (res);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1241,6 +1241,9 @@ decode_utf8(char **sPtr, char *end, char* encoding)
|
|||
#endif
|
||||
}
|
||||
|
||||
/* compiler.transformer.Transformer.decode_literal depends on what
|
||||
might seem like minor details of this function -- changes here
|
||||
must be reflected there. */
|
||||
static PyObject *
|
||||
parsestr(struct compiling *c, char *s)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue