Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-117486: Improve behavior for user-defined AST subclasses (#118212) 

Now, such classes will no longer require changes in Python 3.13 in the normal case.
The test suite for robotframework passes with no DeprecationWarnings under this PR.

I also added a new DeprecationWarning for the case where `_field_types` exists
but is incomplete, since that seems likely to indicate a user mistake.
This commit is contained in:
Jelle Zijlstra 2024-05-06 15:57:27 -07:00 committed by GitHub
parent 040571f258
commit e0422198fb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 94 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Doc/library/ast.rst
View File

@ -61,7 +61,7 @@ Node classes
.. attribute:: _fields
Each concrete class has an attribute :attr:`_fields` which gives the names
Each concrete class has an attribute :attr:`!_fields` which gives the names
of all child nodes.
Each instance of a concrete class has one attribute for each child node,
@ -74,6 +74,18 @@ Node classes
as Python lists. All possible attributes must be present and have valid
values when compiling an AST with :func:`compile`.
.. attribute:: _field_types
The :attr:`!_field_types` attribute on each concrete class is a dictionary
mapping field names (as also listed in :attr:`_fields`) to their types.
.. doctest::
>>> ast.TypeVar._field_types
{'name': <class 'str'>, 'bound': ast.expr | None, 'default_value': ast.expr | None}
.. versionadded:: 3.13
.. attribute:: lineno
col_offset
end_lineno

6
Doc/whatsnew/3.13.rst
View File

@ -384,6 +384,12 @@ ast
argument that does not map to a field on the AST node is now deprecated,
and will raise an exception in Python 3.15.
These changes do not apply to user-defined subclasses of :class:`ast.AST`,
unless the class opts in to the new behavior by setting the attribute
:attr:`ast.AST._field_types`.
(Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.)
* :func:`ast.parse` now accepts an optional argument *optimize*
which is passed on to the :func:`compile` built-in. This makes it
possible to obtain an optimized AST.

41
Lib/test/test_ast.py
View File

@ -3036,7 +3036,7 @@ class ASTConstructorTests(unittest.TestCase):
self.assertEqual(node.name, 'foo')
self.assertEqual(node.decorator_list, [])
def test_custom_subclass(self):
def test_custom_subclass_with_no_fields(self):
class NoInit(ast.AST):
pass
@ -3044,17 +3044,17 @@ class ASTConstructorTests(unittest.TestCase):
self.assertIsInstance(obj, NoInit)
self.assertEqual(obj.__dict__, {})
def test_fields_but_no_field_types(self):
class Fields(ast.AST):
_fields = ('a',)
with self.assertWarnsRegex(DeprecationWarning,
r"Fields provides _fields but not _field_types."):
obj = Fields()
obj = Fields()
with self.assertRaises(AttributeError):
obj.a
obj = Fields(a=1)
self.assertEqual(obj.a, 1)
def test_fields_and_types(self):
class FieldsAndTypes(ast.AST):
_fields = ('a',)
_field_types = {'a': int | None}
@ -3065,6 +3065,7 @@ class ASTConstructorTests(unittest.TestCase):
obj = FieldsAndTypes(a=1)
self.assertEqual(obj.a, 1)
def test_fields_and_types_no_default(self):
class FieldsAndTypesNoDefault(ast.AST):
_fields = ('a',)
_field_types = {'a': int}
@ -3077,6 +3078,38 @@ class ASTConstructorTests(unittest.TestCase):
obj = FieldsAndTypesNoDefault(a=1)
self.assertEqual(obj.a, 1)
def test_incomplete_field_types(self):
class MoreFieldsThanTypes(ast.AST):
_fields = ('a', 'b')
_field_types = {'a': int | None}
a: int | None = None
b: int | None = None
with self.assertWarnsRegex(
DeprecationWarning,
r"Field 'b' is missing from MoreFieldsThanTypes\._field_types"
):
obj = MoreFieldsThanTypes()
self.assertIs(obj.a, None)
self.assertIs(obj.b, None)
obj = MoreFieldsThanTypes(a=1, b=2)
self.assertEqual(obj.a, 1)
self.assertEqual(obj.b, 2)
def test_complete_field_types(self):
class _AllFieldTypes(ast.AST):
_fields = ('a', 'b')
_field_types = {'a': int | None, 'b': list[str]}
# This must be set explicitly
a: int | None = None
# This will add an implicit empty list default
b: list[str]
obj = _AllFieldTypes()
self.assertIs(obj.a, None)
self.assertEqual(obj.b, [])
@support.cpython_only
class ModuleStateTests(unittest.TestCase):

4
Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst Normal file
View File

@ -0,0 +1,4 @@
Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such
classes will now require no changes in the usual case to conform with the
behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle
Zijlstra.

31
Parser/asdl_c.py
View File

@ -979,14 +979,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
goto cleanup;
}
if (field_types == NULL) {
if (PyErr_WarnFormat(
PyExc_DeprecationWarning, 1,
"%.400s provides _fields but not _field_types. "
"This will become an error in Python 3.15.",
Py_TYPE(self)->tp_name
) < 0) {
res = -1;
}
// Probably a user-defined subclass of AST that lacks _field_types.
// This will continue to work as it did before 3.13; i.e., attributes
// that are not passed in simply do not exist on the instance.
goto cleanup;
}
remaining_list = PySequence_List(remaining_fields);
@ -997,12 +992,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
PyObject *name = PyList_GET_ITEM(remaining_list, i);
PyObject *type = PyDict_GetItemWithError(field_types, name);
if (!type) {
if (!PyErr_Occurred()) {
PyErr_SetObject(PyExc_KeyError, name);
if (PyErr_Occurred()) {
goto set_remaining_cleanup;
}
else {
if (PyErr_WarnFormat(
PyExc_DeprecationWarning, 1,
"Field '%U' is missing from %.400s._field_types. "
"This will become an error in Python 3.15.",
name, Py_TYPE(self)->tp_name
) < 0) {
goto set_remaining_cleanup;
}
}
goto set_remaining_cleanup;
}
if (_PyUnion_Check(type)) {
else if (_PyUnion_Check(type)) {
// optional field
// do nothing, we'll have set a None default on the class
}
@ -1026,8 +1030,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
"This will become an error in Python 3.15.",
Py_TYPE(self)->tp_name, name
) < 0) {
res = -1;
goto cleanup;
goto set_remaining_cleanup;
}
}
}

31
Python/Python-ast.c generated
View File

@ -5178,14 +5178,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
goto cleanup;
}
if (field_types == NULL) {
if (PyErr_WarnFormat(
PyExc_DeprecationWarning, 1,
"%.400s provides _fields but not _field_types. "
"This will become an error in Python 3.15.",
Py_TYPE(self)->tp_name
) < 0) {
res = -1;
}
// Probably a user-defined subclass of AST that lacks _field_types.
// This will continue to work as it did before 3.13; i.e., attributes
// that are not passed in simply do not exist on the instance.
goto cleanup;
}
remaining_list = PySequence_List(remaining_fields);
@ -5196,12 +5191,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
PyObject *name = PyList_GET_ITEM(remaining_list, i);
PyObject *type = PyDict_GetItemWithError(field_types, name);
if (!type) {
if (!PyErr_Occurred()) {
PyErr_SetObject(PyExc_KeyError, name);
if (PyErr_Occurred()) {
goto set_remaining_cleanup;
}
else {
if (PyErr_WarnFormat(
PyExc_DeprecationWarning, 1,
"Field '%U' is missing from %.400s._field_types. "
"This will become an error in Python 3.15.",
name, Py_TYPE(self)->tp_name
) < 0) {
goto set_remaining_cleanup;
}
}
goto set_remaining_cleanup;
}
if (_PyUnion_Check(type)) {
else if (_PyUnion_Check(type)) {
// optional field
// do nothing, we'll have set a None default on the class
}
@ -5225,8 +5229,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
"This will become an error in Python 3.15.",
Py_TYPE(self)->tp_name, name
) < 0) {
res = -1;
goto cleanup;
goto set_remaining_cleanup;
}
}
}