From 168660b547d5f683c5d3c60447cfa8c6d620efc3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 Apr 2020 00:47:39 +0100 Subject: [PATCH] bpo-40141: Add line and column information to ast.keyword nodes (GH-19283) --- Include/Python-ast.h | 10 +- Lib/test/test_ast.py | 2 +- .../2020-04-01-21-50-37.bpo-40141.8fCRVj.rst | 2 + Parser/Python.asdl | 1 + Python/Python-ast.c | 103 +++++++++++++++++- Python/ast.c | 7 +- 6 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-04-01-21-50-37.bpo-40141.8fCRVj.rst diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 7db0037bb0d..e7afa1e6579 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -427,6 +427,10 @@ struct _arg { struct _keyword { identifier arg; expr_ty value; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; }; struct _alias { @@ -670,8 +674,10 @@ arguments_ty _Py_arguments(asdl_seq * posonlyargs, asdl_seq * args, arg_ty arg_ty _Py_arg(identifier arg, expr_ty annotation, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) -keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); +#define keyword(a0, a1, a2, a3, a4, a5, a6) _Py_keyword(a0, a1, a2, a3, a4, a5, a6) +keyword_ty _Py_keyword(identifier arg, expr_ty value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); #define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3fd982c79ea..6f86eb954ec 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -2006,7 +2006,7 @@ eval_results = [ ('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), ('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), ('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 7), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), ('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), ('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), ('Expression', ('Constant', (1, 0, 1, 2), 10, None)), diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-01-21-50-37.bpo-40141.8fCRVj.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-01-21-50-37.bpo-40141.8fCRVj.rst new file mode 100644 index 00000000000..c6ea50e2ce8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-01-21-50-37.bpo-40141.8fCRVj.rst @@ -0,0 +1,2 @@ +Add column and line information to ``ast.keyword`` nodes. Patch by Pablo +Galindo. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 11d877d6648..f789f1da456 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -114,6 +114,7 @@ module Python -- keyword arguments supplied to call (NULL identifier for **kwargs) keyword = (identifier? arg, expr value) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) -- import name with optional 'as' alias. alias = (identifier name, identifier? asname) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index c7c7fda45d8..488276a4555 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1060,6 +1060,12 @@ static const char * const arg_fields[]={ "type_comment", }; static PyObject* ast2obj_keyword(void*); +static const char * const keyword_attributes[] = { + "lineno", + "col_offset", + "end_lineno", + "end_col_offset", +}; static const char * const keyword_fields[]={ "arg", "value", @@ -1985,9 +1991,14 @@ static int init_types(void) 2, "keyword(identifier? arg, expr value)"); if (!state->keyword_type) return 0; - if (!add_attributes(state->keyword_type, NULL, 0)) return 0; + if (!add_attributes(state->keyword_type, keyword_attributes, 4)) return 0; if (PyObject_SetAttr(state->keyword_type, state->arg, Py_None) == -1) return 0; + if (PyObject_SetAttr(state->keyword_type, state->end_lineno, Py_None) == -1) + return 0; + if (PyObject_SetAttr(state->keyword_type, state->end_col_offset, Py_None) + == -1) + return 0; state->alias_type = make_type("alias", state->AST_type, alias_fields, 2, "alias(identifier name, identifier? asname)"); if (!state->alias_type) return 0; @@ -3422,7 +3433,8 @@ arg(identifier arg, expr_ty annotation, string type_comment, int lineno, int } keyword_ty -keyword(identifier arg, expr_ty value, PyArena *arena) +keyword(identifier arg, expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { keyword_ty p; if (!value) { @@ -3435,6 +3447,10 @@ keyword(identifier arg, expr_ty value, PyArena *arena) return NULL; p->arg = arg; p->value = value; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; return p; } @@ -4954,6 +4970,27 @@ ast2obj_keyword(void* _o) if (PyObject_SetAttr(result, astmodulestate_global->value, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_int(o->lineno); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->col_offset); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->col_offset, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_lineno); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->end_col_offset); + if (!value) goto failed; + if (PyObject_SetAttr(result, astmodulestate_global->end_col_offset, value) + < 0) + goto failed; + Py_DECREF(value); return result; failed: Py_XDECREF(value); @@ -9628,6 +9665,10 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena) PyObject* tmp = NULL; identifier arg; expr_ty value; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; if (_PyObject_LookupAttr(obj, astmodulestate_global->arg, &tmp) < 0) { return 1; @@ -9655,7 +9696,63 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena) if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = keyword(arg, value, arena); + if (_PyObject_LookupAttr(obj, astmodulestate_global->lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from keyword"); + return 1; + } + else { + int res; + res = obj2ast_int(tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, astmodulestate_global->col_offset, &tmp) < 0) + { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from keyword"); + return 1; + } + else { + int res; + res = obj2ast_int(tmp, &col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, astmodulestate_global->end_lineno, &tmp) < 0) + { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, astmodulestate_global->end_col_offset, &tmp) + < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = keyword(arg, value, lineno, col_offset, end_lineno, end_col_offset, + arena); return 0; failed: Py_XDECREF(tmp); diff --git a/Python/ast.c b/Python/ast.c index fb23c026e8c..550ee03b1ac 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3013,7 +3013,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, e = ast_for_expr(c, CHILD(ch, 1)); if (!e) return NULL; - kw = keyword(NULL, e, c->c_arena); + kw = keyword(NULL, e, chch->n_lineno, chch->n_col_offset, + e->end_lineno, e->end_col_offset, c->c_arena); asdl_seq_SET(keywords, nkeywords++, kw); ndoublestars++; } @@ -3103,7 +3104,9 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, e = ast_for_expr(c, CHILD(ch, 2)); if (!e) return NULL; - kw = keyword(key, e, c->c_arena); + kw = keyword(key, e, chch->n_lineno, chch->n_col_offset, + chch->n_end_lineno, chch->n_end_col_offset, c->c_arena); + if (!kw) return NULL; asdl_seq_SET(keywords, nkeywords++, kw);