gh-105858: Improve AST node constructors (#105880)

Demonstration:

>>> ast.FunctionDef.__annotations__
{'name': <class 'str'>, 'args': <class 'ast.arguments'>, 'body': list[ast.stmt], 'decorator_list': list[ast.expr], 'returns': ast.expr | None, 'type_comment': str | None, 'type_params': list[ast.type_param]}
>>> ast.FunctionDef()
<stdin>:1: DeprecationWarning: FunctionDef.__init__ missing 1 required positional argument: 'name'. This will become an error in Python 3.15.
<stdin>:1: DeprecationWarning: FunctionDef.__init__ missing 1 required positional argument: 'args'. This will become an error in Python 3.15.
<ast.FunctionDef object at 0x101959460>
>>> node = ast.FunctionDef(name="foo", args=ast.arguments())
>>> node.decorator_list
[]
>>> ast.FunctionDef(whatever="you want", name="x", args=ast.arguments())
<stdin>:1: DeprecationWarning: FunctionDef.__init__ got an unexpected keyword argument 'whatever'. Support for arbitrary keyword arguments is deprecated and will be removed in Python 3.15.
<ast.FunctionDef object at 0x1019581f0>

Doc/library/ast.rst

@@ -103,20 +103,15 @@ Node classes
    For example, to create and populate an :class:`ast.UnaryOp` node, you could
    use ::
 
-      node = ast.UnaryOp()
-      node.op = ast.USub()
-      node.operand = ast.Constant()
-      node.operand.value = 5
-      node.operand.lineno = 0
-      node.operand.col_offset = 0
-      node.lineno = 0
-      node.col_offset = 0
-
-   or the more compact ::
-
       node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0),
                          lineno=0, col_offset=0)
 
+   If a field that is optional in the grammar is omitted from the constructor,
+   it defaults to ``None``. If a list field is omitted, it defaults to the empty
+   list. If any other field is omitted, a :exc:`DeprecationWarning` is raised
+   and the AST node will not have this field. In Python 3.15, this condition will
+   raise an error.
+
 .. versionchanged:: 3.8
 
    Class :class:`ast.Constant` is now used for all constants.
@@ -140,6 +135,14 @@ Node classes
    In the meantime, instantiating them will return an instance of
    a different class.
 
+.. deprecated-removed:: 3.13 3.15
+
+   Previous versions of Python allowed the creation of AST nodes that were missing
+   required fields. Similarly, AST node constructors allowed arbitrary keyword
+   arguments that were set as attributes of the AST node, even if they did not
+   match any of the fields of the AST node. This behavior is deprecated and will
+   be removed in Python 3.15.
+
 .. note::
     The descriptions of the specific node classes displayed here
     were initially adapted from the fantastic `Green Tree

Doc/whatsnew/3.13.rst

@@ -206,6 +206,21 @@ array
 ast
 ---
 
+* The constructors of node types in the :mod:`ast` module are now stricter
+  in the arguments they accept, and have more intuitive behaviour when
+  arguments are omitted.
+
+  If an optional field on an AST node is not included as an argument when
+  constructing an instance, the field will now be set to ``None``. Similarly,
+  if a list field is omitted, that field will now be set to an empty list.
+  (Previously, in both cases, the attribute would be missing on the newly
+  constructed AST node instance.)
+
+  If other arguments are omitted, a :exc:`DeprecationWarning` is emitted.
+  This will cause an exception in Python 3.15. Similarly, passing a keyword
+  argument that does not map to a field on the AST node is now deprecated,
+  and will raise an exception in Python 3.15.
+
 * :func:`ast.parse` now accepts an optional argument ``optimize``
   which is passed on to the :func:`compile` built-in. This makes it
   possible to obtain an optimized ``AST``.

Include/internal/pycore_global_objects_fini_generated.h

@@ -753,6 +753,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_dealloc_warn));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_feature_version));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_field_types));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fields_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load));

Include/internal/pycore_global_strings.h

@@ -242,6 +242,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_check_retval_)
         STRUCT_FOR_ID(_dealloc_warn)
         STRUCT_FOR_ID(_feature_version)
+        STRUCT_FOR_ID(_field_types)
         STRUCT_FOR_ID(_fields_)
         STRUCT_FOR_ID(_finalizing)
         STRUCT_FOR_ID(_find_and_load)

Include/internal/pycore_runtime_init_generated.h

@@ -751,6 +751,7 @@ extern "C" {
     INIT_ID(_check_retval_), \
     INIT_ID(_dealloc_warn), \
     INIT_ID(_feature_version), \
+    INIT_ID(_field_types), \
     INIT_ID(_fields_), \
     INIT_ID(_finalizing), \
     INIT_ID(_find_and_load), \

Include/internal/pycore_unicodeobject_generated.h

@@ -567,6 +567,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(_feature_version);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(_field_types);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(_fields_);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);

Lib/test/test_ast.py

@@ -525,17 +525,38 @@ class AST_Tests(unittest.TestCase):
             if name == 'Index':
                 continue
             if self._is_ast_node(name, item):
-                x = item()
+                x = self._construct_ast_class(item)
                 if isinstance(x, ast.AST):
                     self.assertIs(type(x._fields), tuple)
 
+    def _construct_ast_class(self, cls):
+        kwargs = {}
+        for name, typ in cls.__annotations__.items():
+            if typ is str:
+                kwargs[name] = 'capybara'
+            elif typ is int:
+                kwargs[name] = 42
+            elif typ is object:
+                kwargs[name] = b'capybara'
+            elif isinstance(typ, type) and issubclass(typ, ast.AST):
+                kwargs[name] = self._construct_ast_class(typ)
+        return cls(**kwargs)
+
     def test_arguments(self):
         x = ast.arguments()
         self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs',
                                      'kw_defaults', 'kwarg', 'defaults'))
+        self.assertEqual(x.__annotations__, {
+            'posonlyargs': list[ast.arg],
+            'args': list[ast.arg],
+            'vararg': ast.arg | None,
+            'kwonlyargs': list[ast.arg],
+            'kw_defaults': list[ast.expr],
+            'kwarg': ast.arg | None,
+            'defaults': list[ast.expr],
+        })
 
-        with self.assertRaises(AttributeError):
-            x.args
+        self.assertEqual(x.args, [])
         self.assertIsNone(x.vararg)
 
         x = ast.arguments(*range(1, 8))
@@ -551,7 +572,7 @@ class AST_Tests(unittest.TestCase):
         self.assertEqual(x._fields, 666)
 
     def test_field_attr_writable(self):
-        x = ast.Constant()
+        x = ast.Constant(1)
         # We can assign to _fields
         x._fields = 666
         self.assertEqual(x._fields, 666)
@@ -611,15 +632,22 @@ class AST_Tests(unittest.TestCase):
 
         self.assertEqual([str(w.message) for w in wlog], [
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+            "Constant.__init__ missing 1 required positional argument: 'value'. This will become "
+            'an error in Python 3.15.',
             'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
             'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+            "Constant.__init__ missing 1 required positional argument: 'value'. This will become "
+            'an error in Python 3.15.',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
             'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+            "Constant.__init__ got an unexpected keyword argument 'foo'. Support for "
+            'arbitrary keyword arguments is deprecated and will be removed in Python '
+            '3.15.',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
             'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
             'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
@@ -636,7 +664,8 @@ class AST_Tests(unittest.TestCase):
         ])
 
     def test_classattrs(self):
-        x = ast.Constant()
+        with self.assertWarns(DeprecationWarning):
+            x = ast.Constant()
         self.assertEqual(x._fields, ('value', 'kind'))
 
         with self.assertRaises(AttributeError):
@@ -651,7 +680,7 @@ class AST_Tests(unittest.TestCase):
         with self.assertRaises(AttributeError):
             x.foobar
 
-        x = ast.Constant(lineno=2)
+        x = ast.Constant(lineno=2, value=3)
         self.assertEqual(x.lineno, 2)
 
         x = ast.Constant(42, lineno=0)
@@ -662,8 +691,9 @@ class AST_Tests(unittest.TestCase):
         self.assertRaises(TypeError, ast.Constant, 1, None, 2)
         self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0)
 
-        # Arbitrary keyword arguments are supported
-        self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar')
+        # Arbitrary keyword arguments are supported (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar')
 
         with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"):
             ast.Constant(1, value=2)
@@ -815,11 +845,11 @@ class AST_Tests(unittest.TestCase):
         assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes)
         assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant)
         assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis)
-        assertNumDeprecated(self.assertNotIsInstance, Constant(), Num)
-        assertStrDeprecated(self.assertNotIsInstance, Constant(), Str)
-        assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes)
-        assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant)
-        assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis)
+        assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num)
+        assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str)
+        assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes)
+        assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant)
+        assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis)
 
         class S(str): pass
         with assertStrDeprecated():
@@ -888,8 +918,9 @@ class AST_Tests(unittest.TestCase):
         self.assertEqual(x.body, body)
 
     def test_nodeclasses(self):
-        # Zero arguments constructor explicitly allowed
-        x = ast.BinOp()
+        # Zero arguments constructor explicitly allowed (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            x = ast.BinOp()
         self.assertEqual(x._fields, ('left', 'op', 'right'))
 
         # Random attribute allowed too
@@ -927,8 +958,9 @@ class AST_Tests(unittest.TestCase):
         self.assertEqual(x.right, 3)
         self.assertEqual(x.lineno, 0)
 
-        # Random kwargs also allowed
-        x = ast.BinOp(1, 2, 3, foobarbaz=42)
+        # Random kwargs also allowed (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            x = ast.BinOp(1, 2, 3, foobarbaz=42)
         self.assertEqual(x.foobarbaz, 42)
 
     def test_no_fields(self):
@@ -941,8 +973,9 @@ class AST_Tests(unittest.TestCase):
 
         for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
             for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
-                ast2 = pickle.loads(pickle.dumps(ast, protocol))
-                self.assertEqual(to_tuple(ast2), to_tuple(ast))
+                with self.subTest(ast=ast, protocol=protocol):
+                    ast2 = pickle.loads(pickle.dumps(ast, protocol))
+                    self.assertEqual(to_tuple(ast2), to_tuple(ast))
 
     def test_invalid_sum(self):
         pos = dict(lineno=2, col_offset=3)
@@ -1310,8 +1343,9 @@ Module(
             'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, '
             'col_offset=0, end_lineno=1, end_col_offset=5))'
         )
-        src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1)
-        new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None))
+        func = ast.Name('spam', ast.Load())
+        src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func)
+        new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func))
         self.assertIsNone(new.end_lineno)
         self.assertIsNone(new.end_col_offset)
         self.assertEqual(new.lineno, 1)
@@ -1570,15 +1604,15 @@ Module(
         self.assertIn('sleep', ns)
 
     def test_recursion_direct(self):
-        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
         e.operand = e
         with self.assertRaises(RecursionError):
             with support.infinite_recursion():
                 compile(ast.Expression(e), "<test>", "eval")
 
     def test_recursion_indirect(self):
-        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
-        f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+        e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
+        f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
         e.operand = f
         f.operand = e
         with self.assertRaises(RecursionError):
@@ -2866,6 +2900,23 @@ class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase
         self.assertASTTransformation(PrintToLog, code, expected)
 
 
+class ASTConstructorTests(unittest.TestCase):
+    """Test the autogenerated constructors for AST nodes."""
+
+    def test_FunctionDef(self):
+        args = ast.arguments()
+        self.assertEqual(args.args, [])
+        self.assertEqual(args.posonlyargs, [])
+        with self.assertWarnsRegex(DeprecationWarning,
+                                   r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"):
+            node = ast.FunctionDef(args=args)
+        self.assertFalse(hasattr(node, "name"))
+        self.assertEqual(node.decorator_list, [])
+        node = ast.FunctionDef(name='foo', args=args)
+        self.assertEqual(node.name, 'foo')
+        self.assertEqual(node.decorator_list, [])
+
+
 @support.cpython_only
 class ModuleStateTests(unittest.TestCase):
     # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state.

Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst

@@ -0,0 +1,8 @@
+Improve the constructors for :mod:`ast` nodes. Arguments of list types now
+default to an empty list if omitted, and optional fields default to ``None``.
+AST nodes now have an
+``__annotations__`` attribute with the expected types of their attributes.
+Passing unrecognized extra arguments to AST nodes is deprecated and will
+become an error in Python 3.15. Omitting a required argument to an AST node
+is deprecated and will become an error in Python 3.15. Patch by Jelle
+Zijlstra.

Parser/asdl_c.py

@@ -15,6 +15,13 @@ TABSIZE = 4
 MAX_COL = 80
 AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n"
 
+builtin_type_to_c_type = {
+    "identifier": "PyUnicode_Type",
+    "string": "PyUnicode_Type",
+    "int": "PyLong_Type",
+    "constant": "PyBaseObject_Type",
+}
+
 def get_c_type(name):
     """Return a string for the C name of the type.
 
@@ -764,6 +771,67 @@ class PyTypesDeclareVisitor(PickleVisitor):
             self.emit("};",0)
 
 
+class AnnotationsVisitor(PickleVisitor):
+    def visitModule(self, mod):
+        self.file.write(textwrap.dedent('''
+            static int
+            add_ast_annotations(struct ast_state *state)
+            {
+                bool cond;
+        '''))
+        for dfn in mod.dfns:
+            self.visit(dfn)
+        self.file.write(textwrap.dedent('''
+                return 1;
+            }
+        '''))
+
+    def visitProduct(self, prod, name):
+        self.emit_annotations(name, prod.fields)
+
+    def visitSum(self, sum, name):
+        for t in sum.types:
+            self.visitConstructor(t, name)
+
+    def visitConstructor(self, cons, name):
+        self.emit_annotations(cons.name, cons.fields)
+
+    def emit_annotations(self, name, fields):
+        self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1)
+        self.emit(f"if (!{name}_annotations) return 0;", 1)
+        for field in fields:
+            self.emit("{", 1)
+            if field.type in builtin_type_to_c_type:
+                self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2)
+            else:
+                self.emit(f"PyObject *type = state->{field.type}_type;", 2)
+            if field.opt:
+                self.emit("type = _Py_union_type_or(type, Py_None);", 2)
+                self.emit("cond = type != NULL;", 2)
+                self.emit_annotations_error(name, 2)
+            elif field.seq:
+                self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2)
+                self.emit("cond = type != NULL;", 2)
+                self.emit_annotations_error(name, 2)
+            else:
+                self.emit("Py_INCREF(type);", 2)
+            self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2)
+            self.emit("Py_DECREF(type);", 2)
+            self.emit_annotations_error(name, 2)
+            self.emit("}", 1)
+        self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1)
+        self.emit_annotations_error(name, 1)
+        self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1)
+        self.emit_annotations_error(name, 1)
+        self.emit(f"Py_DECREF({name}_annotations);", 1)
+
+    def emit_annotations_error(self, name, depth):
+        self.emit("if (!cond) {", depth)
+        self.emit(f"Py_DECREF({name}_annotations);", depth + 1)
+        self.emit("return 0;", depth + 1)
+        self.emit("}", depth)
+
+
 class PyTypesVisitor(PickleVisitor):
 
     def visitModule(self, mod):
@@ -812,7 +880,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
 
     Py_ssize_t i, numfields = 0;
     int res = -1;
-    PyObject *key, *value, *fields;
+    PyObject *key, *value, *fields, *remaining_fields = NULL;
     if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
         goto cleanup;
     }
@@ -821,6 +889,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
         if (numfields == -1) {
             goto cleanup;
         }
+        remaining_fields = PySet_New(fields);
+    }
+    else {
+        remaining_fields = PySet_New(NULL);
+    }
+    if (remaining_fields == NULL) {
+        goto cleanup;
     }
 
     res = 0; /* if no error occurs, this stays 0 to the end */
@@ -840,6 +915,11 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             goto cleanup;
         }
         res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
+        if (PySet_Discard(remaining_fields, name) < 0) {
+            res = -1;
+            Py_DECREF(name);
+            goto cleanup;
+        }
         Py_DECREF(name);
         if (res < 0) {
             goto cleanup;
@@ -852,13 +932,14 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             if (contains == -1) {
                 res = -1;
                 goto cleanup;
-            } else if (contains == 1) {
-                Py_ssize_t p = PySequence_Index(fields, key);
+            }
+            else if (contains == 1) {
+                int p = PySet_Discard(remaining_fields, key);
                 if (p == -1) {
                     res = -1;
                     goto cleanup;
                 }
-                if (p < PyTuple_GET_SIZE(args)) {
+                if (p == 0) {
                     PyErr_Format(PyExc_TypeError,
                         "%.400s got multiple values for argument '%U'",
                         Py_TYPE(self)->tp_name, key);
@@ -866,15 +947,91 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
                     goto cleanup;
                 }
             }
+            else if (
+                PyUnicode_CompareWithASCIIString(key, "lineno") != 0 &&
+                PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 &&
+                PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 &&
+                PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0
+            ) {
+                if (PyErr_WarnFormat(
+                    PyExc_DeprecationWarning, 1,
+                    "%.400s.__init__ got an unexpected keyword argument '%U'. "
+                    "Support for arbitrary keyword arguments is deprecated "
+                    "and will be removed in Python 3.15.",
+                    Py_TYPE(self)->tp_name, key
+                ) < 0) {
+                    res = -1;
+                    goto cleanup;
+                }
+            }
             res = PyObject_SetAttr(self, key, value);
             if (res < 0) {
                 goto cleanup;
             }
         }
     }
+    Py_ssize_t size = PySet_Size(remaining_fields);
+    PyObject *field_types = NULL, *remaining_list = NULL;
+    if (size > 0) {
+        if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types),
+                                      &field_types)) {
+            res = -1;
+            goto cleanup;
+        }
+        remaining_list = PySequence_List(remaining_fields);
+        if (!remaining_list) {
+            goto set_remaining_cleanup;
+        }
+        for (Py_ssize_t i = 0; i < size; i++) {
+            PyObject *name = PyList_GET_ITEM(remaining_list, i);
+            PyObject *type = PyDict_GetItemWithError(field_types, name);
+            if (!type) {
+                if (!PyErr_Occurred()) {
+                    PyErr_SetObject(PyExc_KeyError, name);
+                }
+                goto set_remaining_cleanup;
+            }
+            if (_PyUnion_Check(type)) {
+                // optional field
+                // do nothing, we'll have set a None default on the class
+            }
+            else if (Py_IS_TYPE(type, &Py_GenericAliasType)) {
+                // list field
+                PyObject *empty = PyList_New(0);
+                if (!empty) {
+                    goto set_remaining_cleanup;
+                }
+                res = PyObject_SetAttr(self, name, empty);
+                Py_DECREF(empty);
+                if (res < 0) {
+                    goto set_remaining_cleanup;
+                }
+            }
+            else {
+                // simple field (e.g., identifier)
+                if (PyErr_WarnFormat(
+                    PyExc_DeprecationWarning, 1,
+                    "%.400s.__init__ missing 1 required positional argument: '%U'. "
+                    "This will become an error in Python 3.15.",
+                    Py_TYPE(self)->tp_name, name
+                ) < 0) {
+                    res = -1;
+                    goto cleanup;
+                }
+            }
+        }
+        Py_DECREF(remaining_list);
+        Py_DECREF(field_types);
+    }
   cleanup:
     Py_XDECREF(fields);
+    Py_XDECREF(remaining_fields);
     return res;
+  set_remaining_cleanup:
+    Py_XDECREF(remaining_list);
+    Py_XDECREF(field_types);
+    res = -1;
+    goto cleanup;
 }
 
 /* Pickling support */
@@ -886,14 +1043,75 @@ ast_type_reduce(PyObject *self, PyObject *unused)
         return NULL;
     }
 
-    PyObject *dict;
+    PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL,
+             *remaining_dict = NULL, *positional_args = NULL;
     if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) {
         return NULL;
     }
+    PyObject *result = NULL;
     if (dict) {
-        return Py_BuildValue("O()N", Py_TYPE(self), dict);
+        // Serialize the fields as positional args if possible, because if we
+        // serialize them as a dict, during unpickling they are set only *after*
+        // the object is constructed, which will now trigger a DeprecationWarning
+        // if the AST type has required fields.
+        if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
+            goto cleanup;
+        }
+        if (fields) {
+            Py_ssize_t numfields = PySequence_Size(fields);
+            if (numfields == -1) {
+                Py_DECREF(dict);
+                goto cleanup;
+            }
+            remaining_dict = PyDict_Copy(dict);
+            Py_DECREF(dict);
+            if (!remaining_dict) {
+                goto cleanup;
+            }
+            positional_args = PyList_New(0);
+            if (!positional_args) {
+                goto cleanup;
+            }
+            for (Py_ssize_t i = 0; i < numfields; i++) {
+                PyObject *name = PySequence_GetItem(fields, i);
+                if (!name) {
+                    goto cleanup;
+                }
+                PyObject *value = PyDict_GetItemWithError(remaining_dict, name);
+                if (!value) {
+                    if (PyErr_Occurred()) {
+                        goto cleanup;
+                    }
+                    break;
+                }
+                if (PyList_Append(positional_args, value) < 0) {
+                    goto cleanup;
+                }
+                if (PyDict_DelItem(remaining_dict, name) < 0) {
+                    goto cleanup;
+                }
+                Py_DECREF(name);
+            }
+            PyObject *args_tuple = PyList_AsTuple(positional_args);
+            if (!args_tuple) {
+                goto cleanup;
+            }
+            result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple,
+                                   remaining_dict);
+        }
+        else {
+            result = Py_BuildValue("O()N", Py_TYPE(self), dict);
+        }
     }
-    return Py_BuildValue("O()", Py_TYPE(self));
+    else {
+        result = Py_BuildValue("O()", Py_TYPE(self));
+    }
+cleanup:
+    Py_XDECREF(fields);
+    Py_XDECREF(remaining_fields);
+    Py_XDECREF(remaining_dict);
+    Py_XDECREF(positional_args);
+    return result;
 }
 
 static PyMemberDef ast_type_members[] = {
@@ -1117,6 +1335,9 @@ static int add_ast_fields(struct ast_state *state)
         for dfn in mod.dfns:
             self.visit(dfn)
         self.file.write(textwrap.dedent('''
+                if (!add_ast_annotations(state)) {
+                    return -1;
+                }
                 return 0;
             }
         '''))
@@ -1534,6 +1755,8 @@ def generate_module_def(mod, metadata, f, internal_h):
         #include "pycore_lock.h"          // _PyOnceFlag
         #include "pycore_interp.h"        // _PyInterpreterState.ast
         #include "pycore_pystate.h"       // _PyInterpreterState_GET()
+        #include "pycore_unionobject.h"   // _Py_union_type_or
+        #include "structmember.h"
         #include <stddef.h>
 
         struct validator {
@@ -1651,6 +1874,7 @@ def write_source(mod, metadata, f, internal_h_file):
     v = ChainOfVisitors(
         SequenceConstructorVisitor(f),
         PyTypesDeclareVisitor(f),
+        AnnotationsVisitor(f),
         PyTypesVisitor(f),
         Obj2ModPrototypeVisitor(f),
         FunctionVisitor(f),