SF patch [ 597919 ] compiler package and SET_LINENO

A variety of changes from Michael Hudson to get the compiler working
with 2.3.  The primary change is the handling of SET_LINENO:

# The set_lineno() function and the explicit emit() calls for
# SET_LINENO below are only used to generate the line number table.
# As of Python 2.3, the interpreter does not have a SET_LINENO
# instruction.  pyassem treats SET_LINENO opcodes as a special case.

A few other small changes:
 - Remove unused code from pycodegen and pyassem.
 - Fix error handling in parsermodule.  When PyParser_SimplerParseString()
   fails, it sets an exception with detailed info.  The parsermodule
   was clobbering that exception and replacing it was a generic
   "could not parse string" exception.  Keep the original exception.

Lib/compiler/pyassem.py

@@ -6,15 +6,8 @@ import sys
 import types
 
 from compiler import misc
-from compiler.consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, \
+from compiler.consts \
-     CO_VARKEYWORDS
+     import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
-
-def xxx_sort(l):
-    l = l[:]
-    def sorter(a, b):
-        return cmp(a.bid, b.bid)
-    l.sort(sorter)
-    return l
 
 class FlowGraph:
     def __init__(self):
@@ -77,7 +70,7 @@ class FlowGraph:
     def emit(self, *inst):
         if self._debug:
             print "\t", inst
-        if inst[0] == 'RETURN_VALUE':
+        if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']:
             self.current.addOutEdge(self.exit)
         if len(inst) == 2 and isinstance(inst[1], Block):
             self.current.addOutEdge(inst[1])
@@ -266,7 +259,7 @@ class Block:
         self.next.append(block)
         assert len(self.next) == 1, map(str, self.next)
 
-    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
+    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE',
                         'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
 
     def pruneNext(self):
@@ -443,7 +436,7 @@ class PyFlowGraph(FlowGraph):
                 insts.append(inst)
                 if len(inst) == 1:
                     pc = pc + 1
-                else:
+                elif inst[0] != "SET_LINENO":
                     # arg takes 2 bytes
                     pc = pc + 3
             end[b] = pc
@@ -452,7 +445,7 @@ class PyFlowGraph(FlowGraph):
             inst = insts[i]
             if len(inst) == 1:
                 pc = pc + 1
-            else:
+            elif inst[0] != "SET_LINENO":
                 pc = pc + 3
             opname = inst[0]
             if self.hasjrel.has_elt(opname):
@@ -580,6 +573,7 @@ class PyFlowGraph(FlowGraph):
                 oparg = t[1]
                 if opname == "SET_LINENO":
                     lnotab.nextLine(oparg)
+                    continue
                 hi, lo = twobyte(oparg)
                 try:
                     lnotab.addCode(self.opnum[opname], lo, hi)
@@ -697,7 +691,7 @@ class LineAddrTable:
             # after the loading of "b".  This works with the C Python
             # compiler because it only generates a SET_LINENO instruction
             # for the assignment.
-            if line > 0:
+            if line >= 0:
                 push = self.lnotab.append
                 while addr > 255:
                     push(255); push(0)
@@ -768,6 +762,7 @@ class StackDepthTracker:
         # PRINT_EXPR?
         'PRINT_ITEM': -1,
         'RETURN_VALUE': -1,
+        'YIELD_VALUE': -1,
         'EXEC_STMT': -3,
         'BUILD_CLASS': -2,
         'STORE_NAME': -1,

Lib/compiler/pycodegen.py

@@ -13,6 +13,7 @@ from compiler.consts import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,\
      CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, CO_FUTURE_DIVISION
 from compiler.pyassem import TupleArg
 
+# XXX The version-specific code can go, since this code only works with 2.x.
 # Do we have Python 1.x or Python 2.x?
 try:
     VERSION = sys.version_info[0]
@@ -32,22 +33,14 @@ EXCEPT = 2
 TRY_FINALLY = 3
 END_FINALLY = 4
 
-# XXX this doesn't seem to be used
-class BlockStack(misc.Stack):
-    __super_init = misc.Stack.__init__
-
-    def __init__(self):
-        self.__super_init(self)
-        self.loop = None
-
 def compileFile(filename, display=0):
-    f = open(filename)
+    f = open(filename, 'U')
     buf = f.read()
     f.close()
     mod = Module(buf, filename)
     try:
         mod.compile(display)
-    except SyntaxError, err:
+    except SyntaxError:
         raise
     else:
         f = open(filename + "c", "wb")
@@ -134,7 +127,7 @@ class Module(AbstractCompileMode):
         # to indicate the type of the value.  simplest way to get the
         # same effect is to call marshal and then skip the code.
         mtime = os.path.getmtime(self.filename)
-        mtime = struct.pack('i', mtime)
+        mtime = struct.pack('<i', mtime)
         return self.MAGIC + mtime
 
 class LocalNameFinder:
@@ -310,9 +303,17 @@ class CodeGenerator:
         else:
             self.emit(prefix + '_NAME', name)
 
-    def set_lineno(self, node, force=0):
+    # The set_lineno() function and the explicit emit() calls for
-        """Emit SET_LINENO if node has lineno attribute and it is
+    # SET_LINENO below are only used to generate the line number table.
-        different than the last lineno emitted.
+    # As of Python 2.3, the interpreter does not have a SET_LINENO
+    # instruction.  pyassem treats SET_LINENO opcodes as a special case.
+
+    def set_lineno(self, node, force=False):
+        """Emit SET_LINENO if necessary.
+
+        The instruction is considered necessary if the node has a
+        lineno attribute and it is different than the last lineno
+        emitted.
 
         Returns true if SET_LINENO was emitted.
 
@@ -326,8 +327,8 @@ class CodeGenerator:
                                    or force):
             self.emit('SET_LINENO', lineno)
             self.last_lineno = lineno
-            return 1
+            return True
-        return 0
+        return False
 
     # The first few visitor methods handle nodes that generator new
     # code objects.  They use class attributes to determine what
@@ -387,9 +388,6 @@ class CodeGenerator:
     def visitClass(self, node):
         gen = self.ClassGen(node, self.scopes,
                             self.get_module())
-        if node.doc:
-            self.emit('LOAD_CONST', node.doc)
-            self.storeName('__doc__')
         walk(node.code, gen)
         gen.finish()
         self.set_lineno(node)
@@ -447,7 +445,7 @@ class CodeGenerator:
         self.nextBlock(loop)
         self.setups.push((LOOP, loop))
 
-        self.set_lineno(node, force=1)
+        self.set_lineno(node, force=True)
         self.visit(node.test)
         self.emit('JUMP_IF_FALSE', else_ or after)
 
@@ -617,7 +615,7 @@ class CodeGenerator:
         return start, anchor
 
     def visitListCompIf(self, node, branch):
-        self.set_lineno(node, force=1)
+        self.set_lineno(node, force=True)
         self.visit(node.test)
         self.emit('JUMP_IF_FALSE', branch)
         self.newBlock()
@@ -975,7 +973,7 @@ class CodeGenerator:
     def visitYield(self, node):
         self.set_lineno(node)
         self.visit(node.value)
-        self.emit('YIELD_STMT')
+        self.emit('YIELD_VALUE')
 
     # slice and subscript stuff
 
@@ -1266,7 +1264,6 @@ class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
         self.__super_init(func, scopes, isLambda, class_name, mod)
         self.graph.setFreeVars(self.scope.get_free_vars())
         self.graph.setCellVars(self.scope.get_cell_vars())
-        if self.graph.checkFlag(CO_GENERATOR_ALLOWED):
         if self.scope.generator is not None:
             self.graph.setFlag(CO_GENERATOR)
 
@@ -1304,6 +1301,12 @@ class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
         self.__super_init(klass, scopes, module)
         self.graph.setFreeVars(self.scope.get_free_vars())
         self.graph.setCellVars(self.scope.get_cell_vars())
+        self.set_lineno(klass)
+        self.emit("LOAD_GLOBAL", "__name__")
+        self.storeName("__module__")
+        if klass.doc:
+            self.emit("LOAD_CONST", klass.doc)
+            self.storeName('__doc__')
 
 def generateArgList(arglist):
     """Generate an arg list marking TupleArgs"""
@@ -1379,7 +1382,5 @@ def wrap_aug(node):
     return wrapper[node.__class__](node)
 
 if __name__ == "__main__":
-    import sys
-
     for file in sys.argv[1:]:
         compileFile(file)

Lib/compiler/symbols.py

@@ -249,6 +249,9 @@ class SymbolVisitor:
         scope = ClassScope(node.name, self.module)
         if parent.nested or isinstance(parent, FunctionScope):
             scope.nested = 1
+        if node.doc is not None:
+            scope.add_def('__doc__')
+        scope.add_def('__module__')
         self.scopes[node] = scope
         prev = self.klass
         self.klass = node.name

Lib/compiler/transformer.py

@@ -37,7 +37,11 @@ from consts import OP_ASSIGN, OP_DELETE, OP_APPLY
 
 def parseFile(path):
     f = open(path)
-    src = f.read()
+    # XXX The parser API tolerates files without a trailing newline,
+    # but not strings without a trailing newline.  Always add an extra
+    # newline to the file contents, since we're going through the string
+    # version of the API.
+    src = f.read() + "\n"
     f.close()
     return parse(src)
 
@@ -100,6 +104,7 @@ class Transformer:
                                token.STRING: self.atom_string,
                                token.NAME: self.atom_name,
                                }
+        self.encoding = None
 
     def transform(self, tree):
         """Transform an AST into a modified parse tree."""
@@ -110,6 +115,7 @@ class Transformer:
     def parsesuite(self, text):
         """Return a modified parse tree for the given suite text."""
         # Hack for handling non-native line endings on non-DOS like OSs.
+        # this can go now we have universal newlines?
         text = text.replace('\x0d', '')
         return self.transform(parser.suite(text))
 
@@ -131,6 +137,12 @@ class Transformer:
     def compile_node(self, node):
         ### emit a line-number node?
         n = node[0]
+
+        if n == symbol.encoding_decl:
+            self.encoding = node[2]
+            node = node[1]
+            n = node[0]
+        
         if n == symbol.single_input:
             return self.single_input(node[1:])
         if n == symbol.file_input:
@@ -519,6 +531,7 @@ class Transformer:
         return self.com_binary(Tuple, nodelist)
 
     testlist_safe = testlist # XXX
+    testlist1 = testlist
     exprlist = testlist
 
     def test(self, nodelist):
@@ -637,11 +650,14 @@ class Transformer:
     def factor(self, nodelist):
         elt = nodelist[0]
         t = elt[0]
+        print "source", nodelist[-1]
         node = self.com_node(nodelist[-1])
+        # need to handle (unary op)constant here...
         if t == token.PLUS:
             node = UnaryAdd(node)
             node.lineno = elt[2]
         elif t == token.MINUS:
+            print node
             node = UnarySub(node)
             node.lineno = elt[2]
         elif t == token.TILDE:
@@ -699,11 +715,21 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
+    def decode_literal(self, lit):
+        if self.encoding:
+            # this is particularly fragile & a bit of a
+            # hack... changes in compile.c:parsestr and
+            # tokenizer.c must be reflected here.
+            if self.encoding not in ['utf-8', 'iso-8859-1']:
+                lit = unicode(lit, 'utf-8').encode(self.encoding)
+            return eval("# coding: %s\n%s" % (self.encoding, lit))
+        else:
+            return eval(lit)
+
     def atom_string(self, nodelist):
-        ### need to verify this matches compile.c
         k = ''
         for node in nodelist:
-            k = k + eval(node[1])
+            k += self.decode_literal(node[1])
         n = Const(k)
         n.lineno = nodelist[0][2]
         return n

Modules/parsermodule.c

@@ -89,7 +89,7 @@ node2tuple(node *n,                     /* node to convert               */
         PyObject *v;
         PyObject *w;
 
-        v = mkseq(1 + NCH(n));
+        v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
         if (v == NULL)
             return (v);
         w = PyInt_FromLong(TYPE(n));
@@ -106,6 +106,9 @@ node2tuple(node *n,                     /* node to convert               */
             }
             (void) addelem(v, i+1, w);
         }
+	
+        if (TYPE(n) == encoding_decl)
+            (void) addelem(v, i+1, PyString_FromString(STR(n)));
         return (v);
     }
     else if (ISTERMINAL(TYPE(n))) {
@@ -478,7 +481,7 @@ err_string(char *message)
 /*  PyObject* parser_do_parse(PyObject* args, int type)
  *
  *  Internal function to actually execute the parse and return the result if
- *  successful, or set an exception if not.
+ *  successful or set an exception if not.
  *
  */
 static PyObject*
@@ -494,10 +497,8 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
                                              (type == PyST_EXPR)
                                              ? eval_input : file_input);
 
-        if (n != 0)
+	if (n)
 	    res = parser_newstobject(n, type);
-        else
-            err_string("could not parse string");
     }
     return (res);
 }

Python/compile.c

@@ -1241,6 +1241,9 @@ decode_utf8(char **sPtr, char *end, char* encoding)
 #endif
 }
 
+/* compiler.transformer.Transformer.decode_literal depends on what 
+   might seem like minor details of this function -- changes here 
+   must be reflected there. */
 static PyObject *
 parsestr(struct compiling *c, char *s)
 {