bpo-39999: Improve compatibility of the ast module. (GH-19056)

* Re-add removed classes Suite, slice, Param, AugLoad and AugStore.
* Add docstrings for dummy classes.
* Add docstrings for attribute aliases.
* Set __module__ to "ast" instead of "_ast".

Doc/whatsnew/3.9.rst

@@ -593,12 +593,19 @@ Deprecated
 
   (Contributed by Victor Stinner in :issue:`39353`.)
 
-* :mod:`ast` classes ``Index`` and ``ExtSlice`` are considered deprecated
+* :mod:`ast` classes ``slice``, ``Index`` and ``ExtSlice`` are considered deprecated
   and will be removed in future Python versions.  ``value`` itself should be
   used instead of ``Index(value)``.  ``Tuple(slices, Load())`` should be
   used instead of ``ExtSlice(slices)``.
   (Contributed by Serhiy Storchaka in :issue:`32892`.)
 
+* :mod:`ast` classes ``Suite``, ``Param``, ``AugLoad`` and ``AugStore``
+  are considered deprecated and will be removed in future Python versions.
+  They were not generated by the parser and not accepted by the code
+  generator in Python 3.
+  (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969`
+  and Serhiy Storchaka in :issue:`39988`.)
+
 * The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized`
   functions are now deprecated and will be removed in Python 3.11. Calling
   :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized
@@ -704,11 +711,6 @@ Removed
   defining ``COUNT_ALLOCS`` macro.
   (Contributed by Victor Stinner in :issue:`39489`.)
 
-* The ``ast.Suite``, ``ast.Param``, ``ast.AugLoad`` and ``ast.AugStore``
-  node classes have been removed due to no longer being needed.
-  (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969`
-  and Serhiy Storchaka in :issue:`39988`.)
-
 
 Porting to Python 3.9
 =====================

Lib/ast.py

@@ -489,6 +489,7 @@ class NodeTransformer(NodeVisitor):
 # It will be removed in future.
 
 def _getter(self):
+    """Deprecated. Use value instead."""
     return self.value
 
 def _setter(self, value):
@@ -499,6 +500,9 @@ Constant.s = property(_getter, _setter)
 
 class _ABC(type):
 
+    def __init__(cls, *args):
+        cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead"""
+
     def __instancecheck__(cls, inst):
         if not isinstance(inst, Constant):
             return False
@@ -564,15 +568,21 @@ _const_node_type_names = {
     type(...): 'Ellipsis',
 }
 
-class Index(AST):
+class slice(AST):
+    """Deprecated AST node class."""
+
+class Index(slice):
+    """Deprecated AST node class. Use the index value directly instead."""
     def __new__(cls, value, **kwargs):
         return value
 
-class ExtSlice(AST):
+class ExtSlice(slice):
+    """Deprecated AST node class. Use ast.Tuple instead."""
     def __new__(cls, dims=(), **kwargs):
         return Tuple(list(dims), Load(), **kwargs)
 
 def _dims_getter(self):
+    """Deprecated. Use elts instead."""
     return self.elts
 
 def _dims_setter(self, value):
@@ -580,6 +590,18 @@ def _dims_setter(self, value):
 
 Tuple.dims = property(_dims_getter, _dims_setter)
 
+class Suite(mod):
+    """Deprecated AST node class.  Unused in Python 3."""
+
+class AugLoad(expr_context):
+    """Deprecated AST node class.  Unused in Python 3."""
+
+class AugStore(expr_context):
+    """Deprecated AST node class.  Unused in Python 3."""
+
+class Param(expr_context):
+    """Deprecated AST node class.  Unused in Python 3."""
+
 
 # Large float and imaginary literals get turned into infinities in the AST.
 # We unparse those infinities to INFSTR.

Lib/test/test_ast.py

@@ -283,7 +283,7 @@ class AST_Tests(unittest.TestCase):
             x.vararg
 
         with self.assertRaises(TypeError):
-            # "_ast.AST constructor takes 0 positional arguments"
+            # "ast.AST constructor takes 0 positional arguments"
             ast.AST(2)
 
     def test_AST_garbage_collection(self):
@@ -573,7 +573,7 @@ class AST_Tests(unittest.TestCase):
         m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
         with self.assertRaises(TypeError) as cm:
             compile(m, "<test>", "exec")
-        self.assertIn("but got <_ast.expr", str(cm.exception))
+        self.assertIn("but got <ast.expr", str(cm.exception))
 
     def test_invalid_identifier(self):
         m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], [])

Misc/NEWS.d/next/Core and Builtins/2020-02-15-15-29-34.bpo-39639.3mqJjm.rst

@@ -1 +0,0 @@
-Remove ``ast.Suite`` node class because it's no longer used. Patch by Batuhan Taskaya.

Misc/NEWS.d/next/Library/2020-02-15-15-29-34.bpo-39639.3mqJjm.rst

@@ -0,0 +1 @@
+Deprecated ``ast.Suite`` node class because it's no longer used. Patch by Batuhan Taskaya.

Misc/NEWS.d/next/Library/2020-03-15-17-56-48.bpo-39969.6snm0c.rst

@@ -1,2 +1,2 @@
-Remove ``ast.Param`` node class because it's no longer used. Patch by
+Deprecated ``ast.Param`` node class because it's no longer used. Patch by
 Batuhan Taskaya.

Misc/NEWS.d/next/Library/2020-03-17-09-35-00.bpo-39988.kXGl35.rst

@@ -1,2 +1,2 @@
-Removed ``ast.AugLoad`` and ``ast.AugStore`` node classes because they are
+Deprecated ``ast.AugLoad`` and ``ast.AugStore`` node classes because they are
 no longer used.

Misc/NEWS.d/next/Library/2020-03-18-11-50-25.bpo-39999.8aOXDT.rst

@@ -0,0 +1,3 @@
+``__module__`` of the AST node classes is now set to "ast" instead of
+"_ast". Added docstrings for dummy AST node classes and deprecated
+attributes.

Parser/asdl_c.py

@@ -774,7 +774,7 @@ static PyType_Slot AST_type_slots[] = {
 };
 
 static PyType_Spec AST_type_spec = {
-    "_ast.AST",
+    "ast.AST",
     sizeof(AST_object),
     0,
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
@@ -800,7 +800,7 @@ make_type(const char *type, PyObject* base, const char* const* fields, int num_f
                     type, base,
                     astmodulestate_global->_fields, fnames,
                     astmodulestate_global->__module__,
-                    astmodulestate_global->_ast,
+                    astmodulestate_global->ast,
                     astmodulestate_global->__doc__, doc);
     Py_DECREF(fnames);
     return result;
@@ -1302,7 +1302,7 @@ def generate_module_def(f, mod):
         visitor_list.add(visitor)
 
     state_strings = {
-        "_ast",
+        "ast",
         "_fields",
         "__doc__",
         "__dict__",

Python/Python-ast.c

@@ -134,7 +134,6 @@ typedef struct {
     PyObject *__dict__;
     PyObject *__doc__;
     PyObject *__module__;
-    PyObject *_ast;
     PyObject *_attributes;
     PyObject *_fields;
     PyObject *alias_type;
@@ -145,6 +144,7 @@ typedef struct {
     PyObject *argtypes;
     PyObject *arguments_type;
     PyObject *asname;
+    PyObject *ast;
     PyObject *attr;
     PyObject *bases;
     PyObject *body;
@@ -354,7 +354,6 @@ static int astmodule_clear(PyObject *module)
     Py_CLEAR(astmodulestate(module)->__dict__);
     Py_CLEAR(astmodulestate(module)->__doc__);
     Py_CLEAR(astmodulestate(module)->__module__);
-    Py_CLEAR(astmodulestate(module)->_ast);
     Py_CLEAR(astmodulestate(module)->_attributes);
     Py_CLEAR(astmodulestate(module)->_fields);
     Py_CLEAR(astmodulestate(module)->alias_type);
@@ -365,6 +364,7 @@ static int astmodule_clear(PyObject *module)
     Py_CLEAR(astmodulestate(module)->argtypes);
     Py_CLEAR(astmodulestate(module)->arguments_type);
     Py_CLEAR(astmodulestate(module)->asname);
+    Py_CLEAR(astmodulestate(module)->ast);
     Py_CLEAR(astmodulestate(module)->attr);
     Py_CLEAR(astmodulestate(module)->bases);
     Py_CLEAR(astmodulestate(module)->body);
@@ -573,7 +573,6 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
     Py_VISIT(astmodulestate(module)->__dict__);
     Py_VISIT(astmodulestate(module)->__doc__);
     Py_VISIT(astmodulestate(module)->__module__);
-    Py_VISIT(astmodulestate(module)->_ast);
     Py_VISIT(astmodulestate(module)->_attributes);
     Py_VISIT(astmodulestate(module)->_fields);
     Py_VISIT(astmodulestate(module)->alias_type);
@@ -584,6 +583,7 @@ static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
     Py_VISIT(astmodulestate(module)->argtypes);
     Py_VISIT(astmodulestate(module)->arguments_type);
     Py_VISIT(astmodulestate(module)->asname);
+    Py_VISIT(astmodulestate(module)->ast);
     Py_VISIT(astmodulestate(module)->attr);
     Py_VISIT(astmodulestate(module)->bases);
     Py_VISIT(astmodulestate(module)->body);
@@ -688,7 +688,6 @@ static int init_identifiers(void)
     if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0;
     if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0;
     if ((state->__module__ = PyUnicode_InternFromString("__module__")) == NULL) return 0;
-    if ((state->_ast = PyUnicode_InternFromString("_ast")) == NULL) return 0;
     if ((state->_attributes = PyUnicode_InternFromString("_attributes")) == NULL) return 0;
     if ((state->_fields = PyUnicode_InternFromString("_fields")) == NULL) return 0;
     if ((state->annotation = PyUnicode_InternFromString("annotation")) == NULL) return 0;
@@ -696,6 +695,7 @@ static int init_identifiers(void)
     if ((state->args = PyUnicode_InternFromString("args")) == NULL) return 0;
     if ((state->argtypes = PyUnicode_InternFromString("argtypes")) == NULL) return 0;
     if ((state->asname = PyUnicode_InternFromString("asname")) == NULL) return 0;
+    if ((state->ast = PyUnicode_InternFromString("ast")) == NULL) return 0;
     if ((state->attr = PyUnicode_InternFromString("attr")) == NULL) return 0;
     if ((state->bases = PyUnicode_InternFromString("bases")) == NULL) return 0;
     if ((state->body = PyUnicode_InternFromString("body")) == NULL) return 0;
@@ -1209,7 +1209,7 @@ static PyType_Slot AST_type_slots[] = {
 };
 
 static PyType_Spec AST_type_spec = {
-    "_ast.AST",
+    "ast.AST",
     sizeof(AST_object),
     0,
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
@@ -1235,7 +1235,7 @@ make_type(const char *type, PyObject* base, const char* const* fields, int num_f
                     type, base,
                     astmodulestate_global->_fields, fnames,
                     astmodulestate_global->__module__,
-                    astmodulestate_global->_ast,
+                    astmodulestate_global->ast,
                     astmodulestate_global->__doc__, doc);
     Py_DECREF(fnames);
     return result;