[Python-checkins] r51489 - in python/trunk: Lib/test/test_parser.py Modules/parsermodule.c
jeremy.hylton
python-checkins at python.org
Tue Aug 22 22:46:01 CEST 2006
Author: jeremy.hylton
Date: Tue Aug 22 22:46:00 2006
New Revision: 51489
Modified:
python/trunk/Lib/test/test_parser.py
python/trunk/Modules/parsermodule.c
Log:
Expose column offset information in parse trees.
Modified: python/trunk/Lib/test/test_parser.py
==============================================================================
--- python/trunk/Lib/test/test_parser.py (original)
+++ python/trunk/Lib/test/test_parser.py Tue Aug 22 22:46:00 2006
@@ -183,6 +183,44 @@
def test_assert(self):
self.check_suite("assert alo < ahi and blo < bhi\n")
+ def test_position(self):
+ # An absolutely minimal test of position information. Better
+ # tests would be a big project.
+ code = "def f(x):\n return x + 1\n"
+ st1 = parser.suite(code)
+ st2 = st1.totuple(line_info=1, col_info=1)
+
+ def walk(tree):
+ node_type = tree[0]
+ next = tree[1]
+ if isinstance(next, tuple):
+ for elt in tree[1:]:
+ for x in walk(elt):
+ yield x
+ else:
+ yield tree
+
+ terminals = list(walk(st2))
+ self.assertEqual([
+ (1, 'def', 1, 0),
+ (1, 'f', 1, 4),
+ (7, '(', 1, 5),
+ (1, 'x', 1, 6),
+ (8, ')', 1, 7),
+ (11, ':', 1, 8),
+ (4, '', 1, 9),
+ (5, '', 2, -1),
+ (1, 'return', 2, 4),
+ (1, 'x', 2, 11),
+ (14, '+', 2, 13),
+ (2, '1', 2, 15),
+ (4, '', 2, 16),
+ (6, '', 2, -1),
+ (4, '', 2, -1),
+ (0, '', 2, -1)],
+ terminals)
+
+
#
# Second, we take *invalid* trees and make sure we get ParserError
# rejections for them.
Modified: python/trunk/Modules/parsermodule.c
==============================================================================
--- python/trunk/Modules/parsermodule.c (original)
+++ python/trunk/Modules/parsermodule.c Tue Aug 22 22:46:00 2006
@@ -74,7 +74,8 @@
node2tuple(node *n, /* node to convert */
SeqMaker mkseq, /* create sequence */
SeqInserter addelem, /* func. to add elem. in seq. */
- int lineno) /* include line numbers? */
+ int lineno, /* include line numbers? */
+ int col_offset) /* include column offsets? */
{
if (n == NULL) {
Py_INCREF(Py_None);
@@ -95,7 +96,7 @@
}
(void) addelem(v, 0, w);
for (i = 0; i < NCH(n); i++) {
- w = node2tuple(CHILD(n, i), mkseq, addelem, lineno);
+ w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset);
if (w == NULL) {
Py_DECREF(v);
return ((PyObject*) NULL);
@@ -108,12 +109,14 @@
return (v);
}
else if (ISTERMINAL(TYPE(n))) {
- PyObject *result = mkseq(2 + lineno);
+ PyObject *result = mkseq(2 + lineno + col_offset);
if (result != NULL) {
(void) addelem(result, 0, PyInt_FromLong(TYPE(n)));
(void) addelem(result, 1, PyString_FromString(STR(n)));
if (lineno == 1)
(void) addelem(result, 2, PyInt_FromLong(n->n_lineno));
+ if (col_offset == 1)
+ (void) addelem(result, 3, PyInt_FromLong(n->n_col_offset));
}
return (result);
}
@@ -289,29 +292,35 @@
parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw)
{
PyObject *line_option = 0;
+ PyObject *col_option = 0;
PyObject *res = 0;
int ok;
- static char *keywords[] = {"ast", "line_info", NULL};
+ static char *keywords[] = {"ast", "line_info", "col_info", NULL};
if (self == NULL) {
- ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2tuple", keywords,
- &PyST_Type, &self, &line_option);
+ ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords,
+ &PyST_Type, &self, &line_option,
+ &col_option);
}
else
- ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:totuple", &keywords[1],
- &line_option);
+ ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1],
+ &line_option, &col_option);
if (ok != 0) {
int lineno = 0;
+ int col_offset = 0;
if (line_option != NULL) {
lineno = (PyObject_IsTrue(line_option) != 0) ? 1 : 0;
}
+ if (col_option != NULL) {
+ col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
+ }
/*
* Convert ST into a tuple representation. Use Guido's function,
* since it's known to work already.
*/
res = node2tuple(((PyST_Object*)self)->st_node,
- PyTuple_New, PyTuple_SetItem, lineno);
+ PyTuple_New, PyTuple_SetItem, lineno, col_offset);
}
return (res);
}
@@ -327,28 +336,34 @@
parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw)
{
PyObject *line_option = 0;
+ PyObject *col_option = 0;
PyObject *res = 0;
int ok;
- static char *keywords[] = {"ast", "line_info", NULL};
+ static char *keywords[] = {"ast", "line_info", "col_info", NULL};
if (self == NULL)
- ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2list", keywords,
- &PyST_Type, &self, &line_option);
+ ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords,
+ &PyST_Type, &self, &line_option,
+ &col_option);
else
- ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:tolist", &keywords[1],
- &line_option);
+ ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1],
+ &line_option, &col_option);
if (ok) {
int lineno = 0;
+ int col_offset = 0;
if (line_option != 0) {
lineno = PyObject_IsTrue(line_option) ? 1 : 0;
}
+ if (col_option != NULL) {
+ col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0;
+ }
/*
* Convert ST into a tuple representation. Use Guido's function,
* since it's known to work already.
*/
res = node2tuple(self->st_node,
- PyList_New, PyList_SetItem, lineno);
+ PyList_New, PyList_SetItem, lineno, col_offset);
}
return (res);
}
More information about the Python-checkins
mailing list