bpo-40939: Remove the old parser (Part 2) (GH-21005)

Remove some remaining files and Makefile targets for the old parser

.gitattributes

@@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf
 
 # Generated files
 # https://github.com/github/linguist#generated-code
-Include/graminit.h          linguist-generated=true
-Python/graminit.h           linguist-generated=true
 Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true

Include/ast.h

@@ -6,19 +6,8 @@ extern "C" {
 #endif
 
 #include "Python-ast.h"   /* mod_ty */
-#include "node.h"         /* node */
 
 PyAPI_FUNC(int) PyAST_Validate(mod_ty);
-PyAPI_FUNC(mod_ty) PyAST_FromNode(
-    const node *n,
-    PyCompilerFlags *flags,
-    const char *filename,       /* decoded from the filesystem encoding */
-    PyArena *arena);
-PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
-    const node *n,
-    PyCompilerFlags *flags,
-    PyObject *filename,
-    PyArena *arena);
 
 /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
 PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty);

Include/bitset.h

@@ -1,23 +0,0 @@
-
-#ifndef Py_BITSET_H
-#define Py_BITSET_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Bitset interface */
-
-#define BYTE            char
-typedef BYTE *bitset;
-
-#define testbit(ss, ibit) (((ss)[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0)
-
-#define BITSPERBYTE     (8*sizeof(BYTE))
-#define BIT2BYTE(ibit)  ((ibit) / BITSPERBYTE)
-#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE)
-#define BIT2MASK(ibit)  (1 << BIT2SHIFT(ibit))
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_BITSET_H */

Include/compile.h

@@ -8,10 +8,6 @@ extern "C" {
 #endif
 
 /* Public interface */
-struct _node; /* Declare the existence of this type */
-PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
-/* XXX (ncoghlan): Unprefixed type name in a public API! */
-
 #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
                    CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
                    CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \

Include/graminit.h

@@ -1,94 +0,0 @@
-/* Generated by Parser/pgen */
-
-#define single_input 256
-#define file_input 257
-#define eval_input 258
-#define decorator 259
-#define decorators 260
-#define decorated 261
-#define async_funcdef 262
-#define funcdef 263
-#define parameters 264
-#define typedargslist 265
-#define tfpdef 266
-#define varargslist 267
-#define vfpdef 268
-#define stmt 269
-#define simple_stmt 270
-#define small_stmt 271
-#define expr_stmt 272
-#define annassign 273
-#define testlist_star_expr 274
-#define augassign 275
-#define del_stmt 276
-#define pass_stmt 277
-#define flow_stmt 278
-#define break_stmt 279
-#define continue_stmt 280
-#define return_stmt 281
-#define yield_stmt 282
-#define raise_stmt 283
-#define import_stmt 284
-#define import_name 285
-#define import_from 286
-#define import_as_name 287
-#define dotted_as_name 288
-#define import_as_names 289
-#define dotted_as_names 290
-#define dotted_name 291
-#define global_stmt 292
-#define nonlocal_stmt 293
-#define assert_stmt 294
-#define compound_stmt 295
-#define async_stmt 296
-#define if_stmt 297
-#define while_stmt 298
-#define for_stmt 299
-#define try_stmt 300
-#define with_stmt 301
-#define with_item 302
-#define except_clause 303
-#define suite 304
-#define namedexpr_test 305
-#define test 306
-#define test_nocond 307
-#define lambdef 308
-#define lambdef_nocond 309
-#define or_test 310
-#define and_test 311
-#define not_test 312
-#define comparison 313
-#define comp_op 314
-#define star_expr 315
-#define expr 316
-#define xor_expr 317
-#define and_expr 318
-#define shift_expr 319
-#define arith_expr 320
-#define term 321
-#define factor 322
-#define power 323
-#define atom_expr 324
-#define atom 325
-#define testlist_comp 326
-#define trailer 327
-#define subscriptlist 328
-#define subscript 329
-#define sliceop 330
-#define exprlist 331
-#define testlist 332
-#define dictorsetmaker 333
-#define classdef 334
-#define arglist 335
-#define argument 336
-#define comp_iter 337
-#define sync_comp_for 338
-#define comp_for 339
-#define comp_if 340
-#define encoding_decl 341
-#define yield_expr 342
-#define yield_arg 343
-#define func_body_suite 344
-#define func_type_input 345
-#define func_type 346
-#define typelist 347

Include/grammar.h

@@ -1,77 +0,0 @@
-
-/* Grammar interface */
-
-#ifndef Py_GRAMMAR_H
-#define Py_GRAMMAR_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "bitset.h" /* Sigh... */
-
-/* A label of an arc */
-
-typedef struct {
-    int          lb_type;
-    const char  *lb_str;
-} label;
-
-#define EMPTY 0         /* Label number 0 is by definition the empty label */
-
-/* A list of labels */
-
-typedef struct {
-    int          ll_nlabels;
-    const label *ll_label;
-} labellist;
-
-/* An arc from one state to another */
-
-typedef struct {
-    short       a_lbl;          /* Label of this arc */
-    short       a_arrow;        /* State where this arc goes to */
-} arc;
-
-/* A state in a DFA */
-
-typedef struct {
-    int          s_narcs;
-    const arc   *s_arc;         /* Array of arcs */
-
-    /* Optional accelerators */
-    int          s_lower;       /* Lowest label index */
-    int          s_upper;       /* Highest label index */
-    int         *s_accel;       /* Accelerator */
-    int          s_accept;      /* Nonzero for accepting state */
-} state;
-
-/* A DFA */
-
-typedef struct {
-    int          d_type;        /* Non-terminal this represents */
-    char        *d_name;        /* For printing */
-    int          d_nstates;
-    state       *d_state;       /* Array of states */
-    bitset       d_first;
-} dfa;
-
-/* A grammar */
-
-typedef struct {
-    int          g_ndfas;
-    const dfa   *g_dfa;         /* Array of DFAs */
-    const labellist g_ll;
-    int          g_start;       /* Start symbol of the grammar */
-    int          g_accel;       /* Set if accelerators present */
-} grammar;
-
-/* FUNCTIONS */
-const dfa *PyGrammar_FindDFA(grammar *g, int type);
-const char *PyGrammar_LabelRepr(label *lb);
-void PyGrammar_AddAccelerators(grammar *g);
-void PyGrammar_RemoveAccelerators(grammar *);
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_GRAMMAR_H */

Include/node.h

@@ -1,47 +0,0 @@
-
-/* Parse tree node interface */
-
-#ifndef Py_NODE_H
-#define Py_NODE_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct _node {
-    short               n_type;
-    char                *n_str;
-    int                 n_lineno;
-    int                 n_col_offset;
-    int                 n_nchildren;
-    struct _node        *n_child;
-    int                 n_end_lineno;
-    int                 n_end_col_offset;
-} node;
-
-PyAPI_FUNC(node *) PyNode_New(int type);
-PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
-                                char *str, int lineno, int col_offset,
-                                int end_lineno, int end_col_offset);
-PyAPI_FUNC(void) PyNode_Free(node *n);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
-#endif
-
-/* Node access functions */
-#define NCH(n)          ((n)->n_nchildren)
-
-#define CHILD(n, i)     (&(n)->n_child[i])
-#define TYPE(n)         ((n)->n_type)
-#define STR(n)          ((n)->n_str)
-#define LINENO(n)       ((n)->n_lineno)
-
-/* Assert that the type of a node is what we expect */
-#define REQ(n, type) assert(TYPE(n) == (type))
-
-PyAPI_FUNC(void) PyNode_ListTree(node *);
-void _PyNode_FinalizeEndPos(node *n);  // helper also used in parsetok.c
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_NODE_H */

Include/parsetok.h

@@ -1,110 +0,0 @@
-/* Parser-tokenizer link interface */
-
-#ifndef Py_LIMITED_API
-#ifndef Py_PARSETOK_H
-#define Py_PARSETOK_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "grammar.h"      /* grammar */
-#include "node.h"         /* node */
-
-typedef struct {
-    int error;
-    PyObject *filename;
-    int lineno;
-    int offset;
-    char *text;                 /* UTF-8-encoded string */
-    int token;
-    int expected;
-} perrdetail;
-
-#if 0
-#define PyPARSE_YIELD_IS_KEYWORD        0x0001
-#endif
-
-#define PyPARSE_DONT_IMPLY_DEDENT       0x0002
-
-#if 0
-#define PyPARSE_WITH_IS_KEYWORD         0x0003
-#define PyPARSE_PRINT_IS_FUNCTION       0x0004
-#define PyPARSE_UNICODE_LITERALS        0x0008
-#endif
-
-#define PyPARSE_IGNORE_COOKIE 0x0010
-#define PyPARSE_BARRY_AS_BDFL 0x0020
-#define PyPARSE_TYPE_COMMENTS 0x0040
-#define PyPARSE_ASYNC_HACKS   0x0080
-
-PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
-                                              perrdetail *);
-PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
-                                             const char *, const char *,
-                                             perrdetail *);
-
-PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
-                                              perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(
-    FILE *fp,
-    const char *filename,       /* decoded from the filesystem encoding */
-    const char *enc,
-    grammar *g,
-    int start,
-    const char *ps1,
-    const char *ps2,
-    perrdetail *err_ret,
-    int flags);
-PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
-    FILE *fp,
-    const char *filename,       /* decoded from the filesystem encoding */
-    const char *enc,
-    grammar *g,
-    int start,
-    const char *ps1,
-    const char *ps2,
-    perrdetail *err_ret,
-    int *flags);
-PyAPI_FUNC(node *) PyParser_ParseFileObject(
-    FILE *fp,
-    PyObject *filename,
-    const char *enc,
-    grammar *g,
-    int start,
-    const char *ps1,
-    const char *ps2,
-    perrdetail *err_ret,
-    int *flags);
-
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(
-    const char *s,
-    const char *filename,       /* decoded from the filesystem encoding */
-    grammar *g,
-    int start,
-    perrdetail *err_ret,
-    int flags);
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
-    const char *s,
-    const char *filename,       /* decoded from the filesystem encoding */
-    grammar *g,
-    int start,
-    perrdetail *err_ret,
-    int *flags);
-PyAPI_FUNC(node *) PyParser_ParseStringObject(
-    const char *s,
-    PyObject *filename,
-    grammar *g,
-    int start,
-    perrdetail *err_ret,
-    int *flags);
-
-/* Note that the following functions are defined in pythonrun.c,
-   not in parsetok.c */
-PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
-PyAPI_FUNC(void) PyParser_ClearError(perrdetail *);
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_PARSETOK_H */
-#endif /* !Py_LIMITED_API */

Lib/symbol.py

@@ -1,122 +0,0 @@
-"""Non-terminal symbols of Python grammar (from "graminit.h")."""
-
-#  This file is automatically generated; please don't muck it up!
-#
-#  To update the symbols in this file, 'cd' to the top directory of
-#  the python source tree after building the interpreter and run:
-#
-#    python3 Tools/scripts/generate_symbol_py.py Include/graminit.h Lib/symbol.py
-#
-# or just
-#
-#    make regen-symbol
-
-import warnings
-
-warnings.warn(
-    "The symbol module is deprecated and will be removed "
-    "in future versions of Python",
-    DeprecationWarning,
-    stacklevel=2,
-)
-
-#--start constants--
-single_input = 256
-file_input = 257
-eval_input = 258
-decorator = 259
-decorators = 260
-decorated = 261
-async_funcdef = 262
-funcdef = 263
-parameters = 264
-typedargslist = 265
-tfpdef = 266
-varargslist = 267
-vfpdef = 268
-stmt = 269
-simple_stmt = 270
-small_stmt = 271
-expr_stmt = 272
-annassign = 273
-testlist_star_expr = 274
-augassign = 275
-del_stmt = 276
-pass_stmt = 277
-flow_stmt = 278
-break_stmt = 279
-continue_stmt = 280
-return_stmt = 281
-yield_stmt = 282
-raise_stmt = 283
-import_stmt = 284
-import_name = 285
-import_from = 286
-import_as_name = 287
-dotted_as_name = 288
-import_as_names = 289
-dotted_as_names = 290
-dotted_name = 291
-global_stmt = 292
-nonlocal_stmt = 293
-assert_stmt = 294
-compound_stmt = 295
-async_stmt = 296
-if_stmt = 297
-while_stmt = 298
-for_stmt = 299
-try_stmt = 300
-with_stmt = 301
-with_item = 302
-except_clause = 303
-suite = 304
-namedexpr_test = 305
-test = 306
-test_nocond = 307
-lambdef = 308
-lambdef_nocond = 309
-or_test = 310
-and_test = 311
-not_test = 312
-comparison = 313
-comp_op = 314
-star_expr = 315
-expr = 316
-xor_expr = 317
-and_expr = 318
-shift_expr = 319
-arith_expr = 320
-term = 321
-factor = 322
-power = 323
-atom_expr = 324
-atom = 325
-testlist_comp = 326
-trailer = 327
-subscriptlist = 328
-subscript = 329
-sliceop = 330
-exprlist = 331
-testlist = 332
-dictorsetmaker = 333
-classdef = 334
-arglist = 335
-argument = 336
-comp_iter = 337
-sync_comp_for = 338
-comp_for = 339
-comp_if = 340
-encoding_decl = 341
-yield_expr = 342
-yield_arg = 343
-func_body_suite = 344
-func_type_input = 345
-func_type = 346
-typelist = 347
-#--end constants--
-
-sym_name = {}
-for _name, _value in list(globals().items()):
-    if type(_value) is type(0):
-        sym_name[_value] = _name
-del _name, _value

Lib/test/test_symbol.py

@@ -1,58 +0,0 @@
-import unittest
-from test import support
-import os
-import sys
-import sysconfig
-import subprocess
-
-
-SYMBOL_FILE              = support.findfile('symbol.py')
-GEN_SYMBOL_FILE          = os.path.join(os.path.dirname(__file__),
-                                        '..', '..', 'Tools', 'scripts',
-                                        'generate_symbol_py.py')
-GRAMMAR_FILE             = os.path.join(os.path.dirname(__file__),
-                                        '..', '..', 'Include', 'graminit.h')
-TEST_PY_FILE             = 'symbol_test.py'
-
-
-class TestSymbolGeneration(unittest.TestCase):
-
-    def _copy_file_without_generated_symbols(self, source_file, dest_file):
-        with open(source_file) as fp:
-            lines = fp.readlines()
-        with open(dest_file, 'w') as fp:
-            fp.writelines(lines[:lines.index("#--start constants--\n") + 1])
-            fp.writelines(lines[lines.index("#--end constants--\n"):])
-
-    def _generate_symbols(self, grammar_file, target_symbol_py_file):
-        proc = subprocess.Popen([sys.executable,
-                                 GEN_SYMBOL_FILE,
-                                 grammar_file,
-                                 target_symbol_py_file], stderr=subprocess.PIPE)
-        stderr = proc.communicate()[1]
-        return proc.returncode, stderr
-
-    def compare_files(self, file1, file2):
-        with open(file1) as fp:
-            lines1 = fp.readlines()
-        with open(file2) as fp:
-            lines2 = fp.readlines()
-        self.assertEqual(lines1, lines2)
-
-    @unittest.skipUnless(sysconfig.is_python_build(),
-                         'test only works from source build directory')
-    def test_real_grammar_and_symbol_file(self):
-        output = support.TESTFN
-        self.addCleanup(support.unlink, output)
-
-        self._copy_file_without_generated_symbols(SYMBOL_FILE, output)
-
-        exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output)
-        self.assertEqual(b'', stderr)
-        self.assertEqual(0, exitcode)
-
-        self.compare_files(SYMBOL_FILE, output)
-
-
-if __name__ == "__main__":
-    unittest.main()

Makefile.pre.in

@@ -344,7 +344,6 @@ PYTHON_OBJS=	\
 		Python/getcopyright.o \
 		Python/getplatform.o \
 		Python/getversion.o \
-		Python/graminit.o \
 		Python/hamt.o \
 		Python/hashtable.o \
 		Python/import.o \
@@ -742,7 +741,7 @@ regen-importlib: Programs/_freeze_importlib
 # Regenerate all generated files
 
 regen-all: regen-opcode regen-opcode-targets regen-typeslots \
-	regen-token regen-symbol regen-ast regen-importlib clinic \
+	regen-token regen-ast regen-importlib clinic \
 	regen-pegen-metaparser regen-pegen
 
 ############################################################################
@@ -881,15 +880,7 @@ regen-keyword:
 		$(srcdir)/Lib/keyword.py.new
 	$(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new
 
-.PHONY: regen-symbol
+Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/Python-ast.h
-regen-symbol: $(srcdir)/Include/graminit.h
-	# Regenerate Lib/symbol.py from Include/graminit.h
-	# using Tools/scripts/generate_symbol_py.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_symbol_py.py \
-		$(srcdir)/Include/graminit.h \
-		$(srcdir)/Lib/symbol.py
-
-Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o Parser/parsetok.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -989,7 +980,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/abstract.h \
 		$(srcdir)/Include/asdl.h \
 		$(srcdir)/Include/ast.h \
-		$(srcdir)/Include/bitset.h \
 		$(srcdir)/Include/bltinmodule.h \
 		$(srcdir)/Include/boolobject.h \
 		$(srcdir)/Include/bytearrayobject.h \
@@ -1027,7 +1017,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/modsupport.h \
 		$(srcdir)/Include/moduleobject.h \
 		$(srcdir)/Include/namespaceobject.h \
-		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
 		$(srcdir)/Include/odictobject.h \

Misc/NEWS.d/next/Core and Builtins/2020-06-20-16-59-02.bpo-40939.6810Ak.rst

@@ -0,0 +1 @@
+Remove the remaining files from the old parser and the :mod:`symbol` module.

PCbuild/pythoncore.vcxproj

@@ -115,7 +115,6 @@
     <ClInclude Include="..\Include\abstract.h" />
     <ClInclude Include="..\Include\asdl.h" />
     <ClInclude Include="..\Include\ast.h" />
-    <ClInclude Include="..\Include\bitset.h" />
     <ClInclude Include="..\Include\boolobject.h" />
     <ClInclude Include="..\Include\bytearrayobject.h" />
     <ClInclude Include="..\Include\bytesobject.h" />
@@ -162,8 +161,6 @@
     <ClInclude Include="..\Include\frameobject.h" />
     <ClInclude Include="..\Include\funcobject.h" />
     <ClInclude Include="..\Include\genobject.h" />
-    <ClInclude Include="..\Include\graminit.h" />
-    <ClInclude Include="..\Include\grammar.h" />
     <ClInclude Include="..\Include\import.h" />
     <ClInclude Include="..\Include\internal\pegen_interface.h" />
     <ClInclude Include="..\Include\internal\pycore_abstract.h" />
@@ -209,14 +206,12 @@
     <ClInclude Include="..\Include\modsupport.h" />
     <ClInclude Include="..\Include\moduleobject.h" />
     <ClInclude Include="..\Include\namespaceobject.h" />
-    <ClInclude Include="..\Include\node.h" />
     <ClInclude Include="..\Include\object.h" />
     <ClInclude Include="..\Include\objimpl.h" />
     <ClInclude Include="..\Include\odictobject.h" />
     <ClInclude Include="..\Include\opcode.h" />
     <ClInclude Include="..\Include\osdefs.h" />
     <ClInclude Include="..\Include\osmodule.h" />
-    <ClInclude Include="..\Include\parsetok.h" />
     <ClInclude Include="..\Include\patchlevel.h" />
     <ClInclude Include="..\Include\picklebufobject.h" />
     <ClInclude Include="..\Include\py_curses.h" />
@@ -451,7 +446,6 @@
     <ClCompile Include="..\Python\getopt.c" />
     <ClCompile Include="..\Python\getplatform.c" />
     <ClCompile Include="..\Python\getversion.c" />
-    <ClCompile Include="..\Python\graminit.c" />
     <ClCompile Include="..\Python\hamt.c" />
     <ClCompile Include="..\Python\hashtable.c" />
     <ClCompile Include="..\Python\import.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -42,9 +42,6 @@
     <ClInclude Include="..\Include\ast.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\bitset.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\boolobject.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -183,12 +180,6 @@
     <ClInclude Include="..\Include\genobject.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\graminit.h">
-      <Filter>Include</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Include\grammar.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\import.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -318,9 +309,6 @@
     <ClInclude Include="..\Include\moduleobject.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\node.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\object.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -336,9 +324,6 @@
     <ClInclude Include="..\Include\osmodule.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\parsetok.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\patchlevel.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -1010,9 +995,6 @@
     <ClCompile Include="..\Python\getversion.c">
       <Filter>Python</Filter>
     </ClCompile>
-    <ClCompile Include="..\Python\graminit.c">
-      <Filter>Python</Filter>
-    </ClCompile>
     <ClCompile Include="..\Python\hamt.h">
       <Filter>Python</Filter>
     </ClCompile>

PCbuild/regen.vcxproj

@@ -133,10 +133,6 @@
     </None>
     <None Include="..\Grammar\Tokens">
     </None>
-    <None Include="..\Include\graminit.h">
-    </None>
-    <None Include="..\Python\graminit.c">
-    </None>
     <None Include="..\Include\token.h">
     </None>
     <None Include="..\Include\opcode.h">
@@ -202,18 +198,12 @@
     </Copy>
     <Warning Text="Keywords updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
   </Target>
-  <Target Name="_RegenSymbols" AfterTargets="_RegenKeywords">
-    <!-- Regenerate Lib/symbol.py from Include/graminit.h using Tools/scripts/generate_symbol_py.py-->
-    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_symbol_py.py &quot;$(PySourcePath)Include\graminit.h&quot;  &quot;$(PySourcePath)Lib\symbol.py&quot;" />
-  </Target>
   <Target Name="_CleanFiles" BeforeTargets="CoreClean">
     <ItemGroup>
       <Clean Include="$(IntDir)keyword.py" />
       <Clean Include="$(IntDir)opcode.h" />
       <Clean Include="$(IntDir)Python-ast.c" />
       <Clean Include="$(IntDir)Python-ast.h" />
-      <Clean Include="$(IntDir)graminit.h.new" />
-      <Clean Include="$(IntDir)graminit.c.new" />
     </ItemGroup>
   </Target>
 </Project>

Parser/node.c

@@ -1,189 +0,0 @@
-/* Parse tree node implementation */
-
-#include "Python.h"
-#include "node.h"
-#include "errcode.h"
-
-node *
-PyNode_New(int type)
-{
-    node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
-    if (n == NULL)
-        return NULL;
-    n->n_type = type;
-    n->n_str = NULL;
-    n->n_lineno = 0;
-    n->n_end_lineno = 0;
-    n->n_col_offset = 0;
-    n->n_end_col_offset = -1;
-    n->n_nchildren = 0;
-    n->n_child = NULL;
-    return n;
-}
-
-/* See comments at XXXROUNDUP below.  Returns -1 on overflow. */
-static int
-fancy_roundup(int n)
-{
-    /* Round up to the closest power of 2 >= n. */
-    int result = 256;
-    assert(n > 128);
-    while (result < n) {
-        result <<= 1;
-        if (result <= 0)
-            return -1;
-    }
-    return result;
-}
-
-/* A gimmick to make massive numbers of reallocs quicker.  The result is
- * a number >= the input.  In PyNode_AddChild, it's used like so, when
- * we're about to add child number current_size + 1:
- *
- *     if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1):
- *         allocate space for XXXROUNDUP(current_size + 1) total children
- *     else:
- *         we already have enough space
- *
- * Since a node starts out empty, we must have
- *
- *     XXXROUNDUP(0) < XXXROUNDUP(1)
- *
- * so that we allocate space for the first child.  One-child nodes are very
- * common (presumably that would change if we used a more abstract form
- * of syntax tree), so to avoid wasting memory it's desirable that
- * XXXROUNDUP(1) == 1.  That in turn forces XXXROUNDUP(0) == 0.
- *
- * Else for 2 <= n <= 128, we round up to the closest multiple of 4.  Why 4?
- * Rounding up to a multiple of an exact power of 2 is very efficient, and
- * most nodes with more than one child have <= 4 kids.
- *
- * Else we call fancy_roundup() to grow proportionately to n.  We've got an
- * extreme case then (like test_longexp.py), and on many platforms doing
- * anything less than proportional growth leads to exorbitant runtime
- * (e.g., MacPython), or extreme fragmentation of user address space (e.g.,
- * Win98).
- *
- * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
- * reported that, with this scheme, 89% of PyObject_REALLOC calls in
- * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually
- * wastes very little memory, but is very effective at sidestepping
- * platform-realloc disasters on vulnerable platforms.
- *
- * Note that this would be straightforward if a node stored its current
- * capacity.  The code is tricky to avoid that.
- */
-#define XXXROUNDUP(n) ((n) <= 1 ? (n) :                         \
-               (n) <= 128 ? (int)_Py_SIZE_ROUND_UP((n), 4) :    \
-               fancy_roundup(n))
-
-
-void
-_PyNode_FinalizeEndPos(node *n)
-{
-    int nch = NCH(n);
-    node *last;
-    if (nch == 0) {
-        return;
-    }
-    last = CHILD(n, nch - 1);
-    _PyNode_FinalizeEndPos(last);
-    n->n_end_lineno = last->n_end_lineno;
-    n->n_end_col_offset = last->n_end_col_offset;
-}
-
-int
-PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset,
-                int end_lineno, int end_col_offset)
-{
-    const int nch = n1->n_nchildren;
-    int current_capacity;
-    int required_capacity;
-    node *n;
-
-    // finalize end position of previous node (if any)
-    if (nch > 0) {
-        _PyNode_FinalizeEndPos(CHILD(n1, nch - 1));
-    }
-
-    if (nch == INT_MAX || nch < 0)
-        return E_OVERFLOW;
-
-    current_capacity = XXXROUNDUP(nch);
-    required_capacity = XXXROUNDUP(nch + 1);
-    if (current_capacity < 0 || required_capacity < 0)
-        return E_OVERFLOW;
-    if (current_capacity < required_capacity) {
-        if ((size_t)required_capacity > SIZE_MAX / sizeof(node)) {
-            return E_NOMEM;
-        }
-        n = n1->n_child;
-        n = (node *) PyObject_REALLOC(n,
-                                      required_capacity * sizeof(node));
-        if (n == NULL)
-            return E_NOMEM;
-        n1->n_child = n;
-    }
-
-    n = &n1->n_child[n1->n_nchildren++];
-    n->n_type = type;
-    n->n_str = str;
-    n->n_lineno = lineno;
-    n->n_col_offset = col_offset;
-    n->n_end_lineno = end_lineno;  // this and below will be updates after all children are added.
-    n->n_end_col_offset = end_col_offset;
-    n->n_nchildren = 0;
-    n->n_child = NULL;
-    return 0;
-}
-
-/* Forward */
-static void freechildren(node *);
-static Py_ssize_t sizeofchildren(node *n);
-
-
-void
-PyNode_Free(node *n)
-{
-    if (n != NULL) {
-        freechildren(n);
-        PyObject_FREE(n);
-    }
-}
-
-Py_ssize_t
-_PyNode_SizeOf(node *n)
-{
-    Py_ssize_t res = 0;
-
-    if (n != NULL)
-        res = sizeof(node) + sizeofchildren(n);
-    return res;
-}
-
-static void
-freechildren(node *n)
-{
-    int i;
-    for (i = NCH(n); --i >= 0; )
-        freechildren(CHILD(n, i));
-    if (n->n_child != NULL)
-        PyObject_FREE(n->n_child);
-    if (STR(n) != NULL)
-        PyObject_FREE(STR(n));
-}
-
-static Py_ssize_t
-sizeofchildren(node *n)
-{
-    Py_ssize_t res = 0;
-    int i;
-    for (i = NCH(n); --i >= 0; )
-        res += sizeofchildren(CHILD(n, i));
-    if (n->n_child != NULL)
-        /* allocated size of n->n_child array */
-        res += XXXROUNDUP(NCH(n)) * sizeof(node);
-    if (STR(n) != NULL)
-        res += strlen(STR(n)) + 1;
-    return res;
-}

Python/ast.c

@@ -5,7 +5,6 @@
  */
 #include "Python.h"
 #include "Python-ast.h"
-#include "node.h"
 #include "ast.h"
 #include "token.h"
 #include "pythonrun.h"

Python/future.c

@@ -1,8 +1,6 @@
 #include "Python.h"
 #include "Python-ast.h"
-#include "node.h"
 #include "token.h"
-#include "graminit.h"
 #include "code.h"
 #include "symtable.h"
 #include "ast.h"

Python/peephole.c

@@ -3,7 +3,6 @@
 #include "Python.h"
 
 #include "Python-ast.h"
-#include "node.h"
 #include "ast.h"
 #include "code.h"
 #include "symtable.h"

Python/pythonrun.c

@@ -20,13 +20,10 @@
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_sysmodule.h"     // _PySys_Audit()
 
-#include "node.h"                 // node
 #include "token.h"                // INDENT
-#include "parsetok.h"             // perrdetail
 #include "errcode.h"              // E_EOF
 #include "code.h"                 // PyCodeObject
 #include "symtable.h"             // PySymtable_BuildObject()
-#include "ast.h"                  // PyAST_FromNodeObject()
 #include "marshal.h"              // PyMarshal_ReadLongFromFile()
 
 #include "pegen_interface.h"      // PyPegen_ASTFrom*

Tools/scripts/generate_symbol_py.py

@@ -1,53 +0,0 @@
-#! /usr/bin/env python3
-# This script generates the symbol.py source file.
-
-import sys
-import re
-
-def main(inFileName="Include/graminit.h", outFileName="Lib/symbol.py"):
-    try:
-        fp = open(inFileName)
-    except OSError as err:
-        sys.stderr.write("I/O error: %s\n" % str(err))
-        sys.exit(1)
-    with fp:
-        lines = fp.read().split("\n")
-    prog = re.compile(
-        "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
-        re.IGNORECASE)
-    tokens = {}
-    for line in lines:
-        match = prog.match(line)
-        if match:
-            name, val = match.group(1, 2)
-            val = int(val)
-            tokens[val] = name          # reverse so we can sort them...
-    keys = sorted(tokens.keys())
-    # load the output skeleton from the target:
-    try:
-        fp = open(outFileName)
-    except OSError as err:
-        sys.stderr.write("I/O error: %s\n" % str(err))
-        sys.exit(2)
-    with fp:
-        format = fp.read().split("\n")
-    try:
-        start = format.index("#--start constants--") + 1
-        end = format.index("#--end constants--")
-    except ValueError:
-        sys.stderr.write("target does not contain format markers")
-        sys.exit(3)
-    lines = []
-    for val in keys:
-        lines.append("%s = %d" % (tokens[val], val))
-    format[start:end] = lines
-    try:
-        fp = open(outFileName, 'w')
-    except OSError as err:
-        sys.stderr.write("I/O error: %s\n" % str(err))
-        sys.exit(4)
-    with fp:
-        fp.write("\n".join(format))
-
-if __name__ == '__main__':
-    main(*sys.argv[1:])

configure

@@ -2739,7 +2739,7 @@ if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then
     # If we're building out-of-tree, we need to make sure the following
     # resources get picked up before their $srcdir counterparts.
     #   Objects/ -> typeslots.inc
-    #   Include/ -> Python-ast.h, graminit.h
+    #   Include/ -> Python-ast.h
     #   Python/  -> importlib.h
     # (A side effect of this is that these resources will automatically be
     #  regenerated when building out-of-tree, regardless of whether or not

configure.ac

@@ -16,7 +16,7 @@ if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then
     # If we're building out-of-tree, we need to make sure the following
     # resources get picked up before their $srcdir counterparts.
     #   Objects/ -> typeslots.inc
-    #   Include/ -> Python-ast.h, graminit.h
+    #   Include/ -> Python-ast.h
     #   Python/  -> importlib.h
     # (A side effect of this is that these resources will automatically be
     #  regenerated when building out-of-tree, regardless of whether or not