Expose column offset information in parse trees.

Lib/test/test_parser.py

@@ -183,6 +183,44 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
     def test_assert(self):
         self.check_suite("assert alo < ahi and blo < bhi\n")
 
+    def test_position(self):
+        # An absolutely minimal test of position information.  Better
+        # tests would be a big project.
+        code = "def f(x):\n    return x + 1\n"
+        st1 = parser.suite(code)
+        st2 = st1.totuple(line_info=1, col_info=1)
+
+        def walk(tree):
+            node_type = tree[0]
+            next = tree[1]
+            if isinstance(next, tuple):
+                for elt in tree[1:]:
+                    for x in walk(elt):
+                        yield x
+            else:
+                yield tree
+            
+        terminals = list(walk(st2))
+        self.assertEqual([
+            (1, 'def', 1, 0),
+            (1, 'f', 1, 4),
+            (7, '(', 1, 5),
+            (1, 'x', 1, 6),
+            (8, ')', 1, 7),
+            (11, ':', 1, 8),
+            (4, '', 1, 9),
+            (5, '', 2, -1),
+            (1, 'return', 2, 4),
+            (1, 'x', 2, 11),
+            (14, '+', 2, 13),
+            (2, '1', 2, 15),
+            (4, '', 2, 16),
+            (6, '', 2, -1),
+            (4, '', 2, -1),
+            (0, '', 2, -1)],
+                         terminals)
+
+
 #
 #  Second, we take *invalid* trees and make sure we get ParserError
 #  rejections for them.

Modules/parsermodule.c

@@ -74,7 +74,8 @@ static PyObject*
 node2tuple(node *n,                     /* node to convert               */
            SeqMaker mkseq,              /* create sequence               */
            SeqInserter addelem,         /* func. to add elem. in seq.    */
-           int lineno)                  /* include line numbers?         */
+           int lineno,                  /* include line numbers?         */
+           int col_offset)              /* include column offsets?       */
 {
     if (n == NULL) {
         Py_INCREF(Py_None);
@@ -95,7 +96,7 @@ node2tuple(node *n,                     /* node to convert               */
         }
         (void) addelem(v, 0, w);
         for (i = 0; i < NCH(n); i++) {
-            w = node2tuple(CHILD(n, i), mkseq, addelem, lineno);
+            w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
             if (w == NULL) {
                 Py_DECREF(v);
                 return ((PyObject*) NULL);
@@ -108,12 +109,14 @@ node2tuple(node *n,                     /* node to convert               */
         return (v);
     }
     else if (ISTERMINAL(TYPE(n))) {
-        PyObject *result = mkseq(2 + lineno);
+        PyObject *result = mkseq(2 + lineno + col_offset);
         if (result != NULL) {
             (void) addelem(result, 0, PyInt_FromLong(TYPE(n)));
             (void) addelem(result, 1, PyString_FromString(STR(n)));
             if (lineno == 1)
                 (void) addelem(result, 2, PyInt_FromLong(n->n_lineno));
+            if (col_offset == 1)
+                (void) addelem(result, 3, PyInt_FromLong(n->n_col_offset));
         }
         return (result);
     }
@@ -289,29 +292,35 @@ static PyObject*
 parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)
 {
     PyObject *line_option = 0;
+    PyObject *col_option = 0;
     PyObject *res = 0;
     int ok;
 
-    static char *keywords[] = {"ast", "line_info", NULL};
+    static char *keywords[] = {"ast", "line_info", "col_info", NULL};
 
     if (self == NULL) {
-        ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2tuple", keywords,
+        ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords,
-                                         &PyST_Type, &self, &line_option);
+                                         &PyST_Type, &self, &line_option,
+                                         &col_option);
     }
     else
-        ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:totuple", &keywords[1],
+        ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1],
-                                         &line_option);
+                                         &line_option, &col_option);
     if (ok != 0) {
         int lineno = 0;
+        int col_offset = 0;
         if (line_option != NULL) {
             lineno = (PyObject_IsTrue(line_option) != 0) ? 1 : 0;
         }
+        if (col_option != NULL) {
+            col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
+        }
         /*
          *  Convert ST into a tuple representation.  Use Guido's function,
          *  since it's known to work already.
          */
         res = node2tuple(((PyST_Object*)self)->st_node,
-                         PyTuple_New, PyTuple_SetItem, lineno);
+                         PyTuple_New, PyTuple_SetItem, lineno, col_offset);
     }
     return (res);
 }
@@ -327,28 +336,34 @@ static PyObject*
 parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)
 {
     PyObject *line_option = 0;
+    PyObject *col_option = 0;
     PyObject *res = 0;
     int ok;
 
-    static char *keywords[] = {"ast", "line_info", NULL};
+    static char *keywords[] = {"ast", "line_info", "col_info", NULL};
 
     if (self == NULL)
-        ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2list", keywords,
+        ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords,
-                                         &PyST_Type, &self, &line_option);
+                                         &PyST_Type, &self, &line_option,
+                                         &col_option);
     else
-        ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:tolist", &keywords[1],
+        ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1],
-                                         &line_option);
+                                         &line_option, &col_option);
     if (ok) {
         int lineno = 0;
+        int col_offset = 0;
         if (line_option != 0) {
             lineno = PyObject_IsTrue(line_option) ? 1 : 0;
         }
+        if (col_option != NULL) {
+            col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
+        }
         /*
          *  Convert ST into a tuple representation.  Use Guido's function,
          *  since it's known to work already.
          */
         res = node2tuple(self->st_node,
-                         PyList_New, PyList_SetItem, lineno);
+                         PyList_New, PyList_SetItem, lineno, col_offset);
     }
     return (res);
 }