Expose column offset information in parse trees.

This commit is contained in:
Jeremy Hylton 2006-08-22 20:46:00 +00:00
parent 670f875a7c
commit 60e96f666c
2 changed files with 68 additions and 15 deletions

View File

@ -183,6 +183,44 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
def test_assert(self): def test_assert(self):
self.check_suite("assert alo < ahi and blo < bhi\n") self.check_suite("assert alo < ahi and blo < bhi\n")
def test_position(self):
# An absolutely minimal test of position information. Better
# tests would be a big project.
code = "def f(x):\n return x + 1\n"
st1 = parser.suite(code)
st2 = st1.totuple(line_info=1, col_info=1)
def walk(tree):
node_type = tree[0]
next = tree[1]
if isinstance(next, tuple):
for elt in tree[1:]:
for x in walk(elt):
yield x
else:
yield tree
terminals = list(walk(st2))
self.assertEqual([
(1, 'def', 1, 0),
(1, 'f', 1, 4),
(7, '(', 1, 5),
(1, 'x', 1, 6),
(8, ')', 1, 7),
(11, ':', 1, 8),
(4, '', 1, 9),
(5, '', 2, -1),
(1, 'return', 2, 4),
(1, 'x', 2, 11),
(14, '+', 2, 13),
(2, '1', 2, 15),
(4, '', 2, 16),
(6, '', 2, -1),
(4, '', 2, -1),
(0, '', 2, -1)],
terminals)
# #
# Second, we take *invalid* trees and make sure we get ParserError # Second, we take *invalid* trees and make sure we get ParserError
# rejections for them. # rejections for them.

View File

@ -74,7 +74,8 @@ static PyObject*
node2tuple(node *n, /* node to convert */ node2tuple(node *n, /* node to convert */
SeqMaker mkseq, /* create sequence */ SeqMaker mkseq, /* create sequence */
SeqInserter addelem, /* func. to add elem. in seq. */ SeqInserter addelem, /* func. to add elem. in seq. */
int lineno) /* include line numbers? */ int lineno, /* include line numbers? */
int col_offset) /* include column offsets? */
{ {
if (n == NULL) { if (n == NULL) {
Py_INCREF(Py_None); Py_INCREF(Py_None);
@ -95,7 +96,7 @@ node2tuple(node *n, /* node to convert */
} }
(void) addelem(v, 0, w); (void) addelem(v, 0, w);
for (i = 0; i < NCH(n); i++) { for (i = 0; i < NCH(n); i++) {
w = node2tuple(CHILD(n, i), mkseq, addelem, lineno); w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
if (w == NULL) { if (w == NULL) {
Py_DECREF(v); Py_DECREF(v);
return ((PyObject*) NULL); return ((PyObject*) NULL);
@ -108,12 +109,14 @@ node2tuple(node *n, /* node to convert */
return (v); return (v);
} }
else if (ISTERMINAL(TYPE(n))) { else if (ISTERMINAL(TYPE(n))) {
PyObject *result = mkseq(2 + lineno); PyObject *result = mkseq(2 + lineno + col_offset);
if (result != NULL) { if (result != NULL) {
(void) addelem(result, 0, PyInt_FromLong(TYPE(n))); (void) addelem(result, 0, PyInt_FromLong(TYPE(n)));
(void) addelem(result, 1, PyString_FromString(STR(n))); (void) addelem(result, 1, PyString_FromString(STR(n)));
if (lineno == 1) if (lineno == 1)
(void) addelem(result, 2, PyInt_FromLong(n->n_lineno)); (void) addelem(result, 2, PyInt_FromLong(n->n_lineno));
if (col_offset == 1)
(void) addelem(result, 3, PyInt_FromLong(n->n_col_offset));
} }
return (result); return (result);
} }
@ -289,29 +292,35 @@ static PyObject*
parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)
{ {
PyObject *line_option = 0; PyObject *line_option = 0;
PyObject *col_option = 0;
PyObject *res = 0; PyObject *res = 0;
int ok; int ok;
static char *keywords[] = {"ast", "line_info", NULL}; static char *keywords[] = {"ast", "line_info", "col_info", NULL};
if (self == NULL) { if (self == NULL) {
ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2tuple", keywords, ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords,
&PyST_Type, &self, &line_option); &PyST_Type, &self, &line_option,
&col_option);
} }
else else
ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:totuple", &keywords[1], ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1],
&line_option); &line_option, &col_option);
if (ok != 0) { if (ok != 0) {
int lineno = 0; int lineno = 0;
int col_offset = 0;
if (line_option != NULL) { if (line_option != NULL) {
lineno = (PyObject_IsTrue(line_option) != 0) ? 1 : 0; lineno = (PyObject_IsTrue(line_option) != 0) ? 1 : 0;
} }
if (col_option != NULL) {
col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
}
/* /*
* Convert ST into a tuple representation. Use Guido's function, * Convert ST into a tuple representation. Use Guido's function,
* since it's known to work already. * since it's known to work already.
*/ */
res = node2tuple(((PyST_Object*)self)->st_node, res = node2tuple(((PyST_Object*)self)->st_node,
PyTuple_New, PyTuple_SetItem, lineno); PyTuple_New, PyTuple_SetItem, lineno, col_offset);
} }
return (res); return (res);
} }
@ -327,28 +336,34 @@ static PyObject*
parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)
{ {
PyObject *line_option = 0; PyObject *line_option = 0;
PyObject *col_option = 0;
PyObject *res = 0; PyObject *res = 0;
int ok; int ok;
static char *keywords[] = {"ast", "line_info", NULL}; static char *keywords[] = {"ast", "line_info", "col_info", NULL};
if (self == NULL) if (self == NULL)
ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2list", keywords, ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords,
&PyST_Type, &self, &line_option); &PyST_Type, &self, &line_option,
&col_option);
else else
ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:tolist", &keywords[1], ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1],
&line_option); &line_option, &col_option);
if (ok) { if (ok) {
int lineno = 0; int lineno = 0;
int col_offset = 0;
if (line_option != 0) { if (line_option != 0) {
lineno = PyObject_IsTrue(line_option) ? 1 : 0; lineno = PyObject_IsTrue(line_option) ? 1 : 0;
} }
if (col_option != NULL) {
col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
}
/* /*
* Convert ST into a tuple representation. Use Guido's function, * Convert ST into a tuple representation. Use Guido's function,
* since it's known to work already. * since it's known to work already.
*/ */
res = node2tuple(self->st_node, res = node2tuple(self->st_node,
PyList_New, PyList_SetItem, lineno); PyList_New, PyList_SetItem, lineno, col_offset);
} }
return (res); return (res);
} }