bpo-40939: Remove the old parser (Part 2) (GH-21005)
Remove some remaining files and Makefile targets for the old parser
This commit is contained in:
Lysandros Nikolaou
GitHub
parent
55460ee6dc
commit
314858e276
|
|
@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf
|
|||
|
||||
# Generated files
|
||||
# https://github.com/github/linguist#generated-code
|
||||
Include/graminit.h linguist-generated=true
|
||||
Python/graminit.h linguist-generated=true
|
||||
Modules/clinic/*.h linguist-generated=true
|
||||
Objects/clinic/*.h linguist-generated=true
|
||||
PC/clinic/*.h linguist-generated=true
|
||||
|
|
|
|||
|
|
@ -6,19 +6,8 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
#include "Python-ast.h" /* mod_ty */
|
||||
#include "node.h" /* node */
|
||||
|
||||
PyAPI_FUNC(int) PyAST_Validate(mod_ty);
|
||||
PyAPI_FUNC(mod_ty) PyAST_FromNode(
|
||||
const node *n,
|
||||
PyCompilerFlags *flags,
|
||||
const char *filename, /* decoded from the filesystem encoding */
|
||||
PyArena *arena);
|
||||
PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
|
||||
const node *n,
|
||||
PyCompilerFlags *flags,
|
||||
PyObject *filename,
|
||||
PyArena *arena);
|
||||
|
||||
/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
|
||||
PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty);
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
|
||||
#ifndef Py_BITSET_H
|
||||
#define Py_BITSET_H
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Bitset interface */
|
||||
|
||||
#define BYTE char
|
||||
typedef BYTE *bitset;
|
||||
|
||||
#define testbit(ss, ibit) (((ss)[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0)
|
||||
|
||||
#define BITSPERBYTE (8*sizeof(BYTE))
|
||||
#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE)
|
||||
#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE)
|
||||
#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit))
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* !Py_BITSET_H */
|
||||
|
|
@ -8,10 +8,6 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
/* Public interface */
|
||||
struct _node; /* Declare the existence of this type */
|
||||
PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
|
||||
/* XXX (ncoghlan): Unprefixed type name in a public API! */
|
||||
|
||||
#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
|
||||
CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
|
||||
CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
/* Generated by Parser/pgen */
|
||||
|
||||
#define single_input 256
|
||||
#define file_input 257
|
||||
#define eval_input 258
|
||||
#define decorator 259
|
||||
#define decorators 260
|
||||
#define decorated 261
|
||||
#define async_funcdef 262
|
||||
#define funcdef 263
|
||||
#define parameters 264
|
||||
#define typedargslist 265
|
||||
#define tfpdef 266
|
||||
#define varargslist 267
|
||||
#define vfpdef 268
|
||||
#define stmt 269
|
||||
#define simple_stmt 270
|
||||
#define small_stmt 271
|
||||
#define expr_stmt 272
|
||||
#define annassign 273
|
||||
#define testlist_star_expr 274
|
||||
#define augassign 275
|
||||
#define del_stmt 276
|
||||
#define pass_stmt 277
|
||||
#define flow_stmt 278
|
||||
#define break_stmt 279
|
||||
#define continue_stmt 280
|
||||
#define return_stmt 281
|
||||
#define yield_stmt 282
|
||||
#define raise_stmt 283
|
||||
#define import_stmt 284
|
||||
#define import_name 285
|
||||
#define import_from 286
|
||||
#define import_as_name 287
|
||||
#define dotted_as_name 288
|
||||
#define import_as_names 289
|
||||
#define dotted_as_names 290
|
||||
#define dotted_name 291
|
||||
#define global_stmt 292
|
||||
#define nonlocal_stmt 293
|
||||
#define assert_stmt 294
|
||||
#define compound_stmt 295
|
||||
#define async_stmt 296
|
||||
#define if_stmt 297
|
||||
#define while_stmt 298
|
||||
#define for_stmt 299
|
||||
#define try_stmt 300
|
||||
#define with_stmt 301
|
||||
#define with_item 302
|
||||
#define except_clause 303
|
||||
#define suite 304
|
||||
#define namedexpr_test 305
|
||||
#define test 306
|
||||
#define test_nocond 307
|
||||
#define lambdef 308
|
||||
#define lambdef_nocond 309
|
||||
#define or_test 310
|
||||
#define and_test 311
|
||||
#define not_test 312
|
||||
#define comparison 313
|
||||
#define comp_op 314
|
||||
#define star_expr 315
|
||||
#define expr 316
|
||||
#define xor_expr 317
|
||||
#define and_expr 318
|
||||
#define shift_expr 319
|
||||
#define arith_expr 320
|
||||
#define term 321
|
||||
#define factor 322
|
||||
#define power 323
|
||||
#define atom_expr 324
|
||||
#define atom 325
|
||||
#define testlist_comp 326
|
||||
#define trailer 327
|
||||
#define subscriptlist 328
|
||||
#define subscript 329
|
||||
#define sliceop 330
|
||||
#define exprlist 331
|
||||
#define testlist 332
|
||||
#define dictorsetmaker 333
|
||||
#define classdef 334
|
||||
#define arglist 335
|
||||
#define argument 336
|
||||
#define comp_iter 337
|
||||
#define sync_comp_for 338
|
||||
#define comp_for 339
|
||||
#define comp_if 340
|
||||
#define encoding_decl 341
|
||||
#define yield_expr 342
|
||||
#define yield_arg 343
|
||||
#define func_body_suite 344
|
||||
#define func_type_input 345
|
||||
#define func_type 346
|
||||
#define typelist 347
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
|
||||
/* Grammar interface */
|
||||
|
||||
#ifndef Py_GRAMMAR_H
|
||||
#define Py_GRAMMAR_H
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "bitset.h" /* Sigh... */
|
||||
|
||||
/* A label of an arc */
|
||||
|
||||
typedef struct {
|
||||
int lb_type;
|
||||
const char *lb_str;
|
||||
} label;
|
||||
|
||||
#define EMPTY 0 /* Label number 0 is by definition the empty label */
|
||||
|
||||
/* A list of labels */
|
||||
|
||||
typedef struct {
|
||||
int ll_nlabels;
|
||||
const label *ll_label;
|
||||
} labellist;
|
||||
|
||||
/* An arc from one state to another */
|
||||
|
||||
typedef struct {
|
||||
short a_lbl; /* Label of this arc */
|
||||
short a_arrow; /* State where this arc goes to */
|
||||
} arc;
|
||||
|
||||
/* A state in a DFA */
|
||||
|
||||
typedef struct {
|
||||
int s_narcs;
|
||||
const arc *s_arc; /* Array of arcs */
|
||||
|
||||
/* Optional accelerators */
|
||||
int s_lower; /* Lowest label index */
|
||||
int s_upper; /* Highest label index */
|
||||
int *s_accel; /* Accelerator */
|
||||
int s_accept; /* Nonzero for accepting state */
|
||||
} state;
|
||||
|
||||
/* A DFA */
|
||||
|
||||
typedef struct {
|
||||
int d_type; /* Non-terminal this represents */
|
||||
char *d_name; /* For printing */
|
||||
int d_nstates;
|
||||
state *d_state; /* Array of states */
|
||||
bitset d_first;
|
||||
} dfa;
|
||||
|
||||
/* A grammar */
|
||||
|
||||
typedef struct {
|
||||
int g_ndfas;
|
||||
const dfa *g_dfa; /* Array of DFAs */
|
||||
const labellist g_ll;
|
||||
int g_start; /* Start symbol of the grammar */
|
||||
int g_accel; /* Set if accelerators present */
|
||||
} grammar;
|
||||
|
||||
/* FUNCTIONS */
|
||||
const dfa *PyGrammar_FindDFA(grammar *g, int type);
|
||||
const char *PyGrammar_LabelRepr(label *lb);
|
||||
void PyGrammar_AddAccelerators(grammar *g);
|
||||
void PyGrammar_RemoveAccelerators(grammar *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* !Py_GRAMMAR_H */
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
|
||||
/* Parse tree node interface */
|
||||
|
||||
#ifndef Py_NODE_H
|
||||
#define Py_NODE_H
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct _node {
|
||||
short n_type;
|
||||
char *n_str;
|
||||
int n_lineno;
|
||||
int n_col_offset;
|
||||
int n_nchildren;
|
||||
struct _node *n_child;
|
||||
int n_end_lineno;
|
||||
int n_end_col_offset;
|
||||
} node;
|
||||
|
||||
PyAPI_FUNC(node *) PyNode_New(int type);
|
||||
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
|
||||
char *str, int lineno, int col_offset,
|
||||
int end_lineno, int end_col_offset);
|
||||
PyAPI_FUNC(void) PyNode_Free(node *n);
|
||||
#ifndef Py_LIMITED_API
|
||||
PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
|
||||
#endif
|
||||
|
||||
/* Node access functions */
|
||||
#define NCH(n) ((n)->n_nchildren)
|
||||
|
||||
#define CHILD(n, i) (&(n)->n_child[i])
|
||||
#define TYPE(n) ((n)->n_type)
|
||||
#define STR(n) ((n)->n_str)
|
||||
#define LINENO(n) ((n)->n_lineno)
|
||||
|
||||
/* Assert that the type of a node is what we expect */
|
||||
#define REQ(n, type) assert(TYPE(n) == (type))
|
||||
|
||||
PyAPI_FUNC(void) PyNode_ListTree(node *);
|
||||
void _PyNode_FinalizeEndPos(node *n); // helper also used in parsetok.c
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* !Py_NODE_H */
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
/* Parser-tokenizer link interface */
|
||||
|
||||
#ifndef Py_LIMITED_API
|
||||
#ifndef Py_PARSETOK_H
|
||||
#define Py_PARSETOK_H
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "grammar.h" /* grammar */
|
||||
#include "node.h" /* node */
|
||||
|
||||
typedef struct {
|
||||
int error;
|
||||
PyObject *filename;
|
||||
int lineno;
|
||||
int offset;
|
||||
char *text; /* UTF-8-encoded string */
|
||||
int token;
|
||||
int expected;
|
||||
} perrdetail;
|
||||
|
||||
#if 0
|
||||
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
|
||||
#endif
|
||||
|
||||
#define PyPARSE_DONT_IMPLY_DEDENT 0x0002
|
||||
|
||||
#if 0
|
||||
#define PyPARSE_WITH_IS_KEYWORD 0x0003
|
||||
#define PyPARSE_PRINT_IS_FUNCTION 0x0004
|
||||
#define PyPARSE_UNICODE_LITERALS 0x0008
|
||||
#endif
|
||||
|
||||
#define PyPARSE_IGNORE_COOKIE 0x0010
|
||||
#define PyPARSE_BARRY_AS_BDFL 0x0020
|
||||
#define PyPARSE_TYPE_COMMENTS 0x0040
|
||||
#define PyPARSE_ASYNC_HACKS 0x0080
|
||||
|
||||
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
|
||||
perrdetail *);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
|
||||
const char *, const char *,
|
||||
perrdetail *);
|
||||
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
|
||||
perrdetail *, int);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileFlags(
|
||||
FILE *fp,
|
||||
const char *filename, /* decoded from the filesystem encoding */
|
||||
const char *enc,
|
||||
grammar *g,
|
||||
int start,
|
||||
const char *ps1,
|
||||
const char *ps2,
|
||||
perrdetail *err_ret,
|
||||
int flags);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
|
||||
FILE *fp,
|
||||
const char *filename, /* decoded from the filesystem encoding */
|
||||
const char *enc,
|
||||
grammar *g,
|
||||
int start,
|
||||
const char *ps1,
|
||||
const char *ps2,
|
||||
perrdetail *err_ret,
|
||||
int *flags);
|
||||
PyAPI_FUNC(node *) PyParser_ParseFileObject(
|
||||
FILE *fp,
|
||||
PyObject *filename,
|
||||
const char *enc,
|
||||
grammar *g,
|
||||
int start,
|
||||
const char *ps1,
|
||||
const char *ps2,
|
||||
perrdetail *err_ret,
|
||||
int *flags);
|
||||
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(
|
||||
const char *s,
|
||||
const char *filename, /* decoded from the filesystem encoding */
|
||||
grammar *g,
|
||||
int start,
|
||||
perrdetail *err_ret,
|
||||
int flags);
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
|
||||
const char *s,
|
||||
const char *filename, /* decoded from the filesystem encoding */
|
||||
grammar *g,
|
||||
int start,
|
||||
perrdetail *err_ret,
|
||||
int *flags);
|
||||
PyAPI_FUNC(node *) PyParser_ParseStringObject(
|
||||
const char *s,
|
||||
PyObject *filename,
|
||||
grammar *g,
|
||||
int start,
|
||||
perrdetail *err_ret,
|
||||
int *flags);
|
||||
|
||||
/* Note that the following functions are defined in pythonrun.c,
|
||||
not in parsetok.c */
|
||||
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
|
||||
PyAPI_FUNC(void) PyParser_ClearError(perrdetail *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif /* !Py_PARSETOK_H */
|
||||
#endif /* !Py_LIMITED_API */
|
||||
122
Lib/symbol.py
122
Lib/symbol.py
|
|
@ -1,122 +0,0 @@
|
|||
"""Non-terminal symbols of Python grammar (from "graminit.h")."""
|
||||
|
||||
# This file is automatically generated; please don't muck it up!
|
||||
#
|
||||
# To update the symbols in this file, 'cd' to the top directory of
|
||||
# the python source tree after building the interpreter and run:
|
||||
#
|
||||
# python3 Tools/scripts/generate_symbol_py.py Include/graminit.h Lib/symbol.py
|
||||
#
|
||||
# or just
|
||||
#
|
||||
# make regen-symbol
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"The symbol module is deprecated and will be removed "
|
||||
"in future versions of Python",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
#--start constants--
|
||||
single_input = 256
|
||||
file_input = 257
|
||||
eval_input = 258
|
||||
decorator = 259
|
||||
decorators = 260
|
||||
decorated = 261
|
||||
async_funcdef = 262
|
||||
funcdef = 263
|
||||
parameters = 264
|
||||
typedargslist = 265
|
||||
tfpdef = 266
|
||||
varargslist = 267
|
||||
vfpdef = 268
|
||||
stmt = 269
|
||||
simple_stmt = 270
|
||||
small_stmt = 271
|
||||
expr_stmt = 272
|
||||
annassign = 273
|
||||
testlist_star_expr = 274
|
||||
augassign = 275
|
||||
del_stmt = 276
|
||||
pass_stmt = 277
|
||||
flow_stmt = 278
|
||||
break_stmt = 279
|
||||
continue_stmt = 280
|
||||
return_stmt = 281
|
||||
yield_stmt = 282
|
||||
raise_stmt = 283
|
||||
import_stmt = 284
|
||||
import_name = 285
|
||||
import_from = 286
|
||||
import_as_name = 287
|
||||
dotted_as_name = 288
|
||||
import_as_names = 289
|
||||
dotted_as_names = 290
|
||||
dotted_name = 291
|
||||
global_stmt = 292
|
||||
nonlocal_stmt = 293
|
||||
assert_stmt = 294
|
||||
compound_stmt = 295
|
||||
async_stmt = 296
|
||||
if_stmt = 297
|
||||
while_stmt = 298
|
||||
for_stmt = 299
|
||||
try_stmt = 300
|
||||
with_stmt = 301
|
||||
with_item = 302
|
||||
except_clause = 303
|
||||
suite = 304
|
||||
namedexpr_test = 305
|
||||
test = 306
|
||||
test_nocond = 307
|
||||
lambdef = 308
|
||||
lambdef_nocond = 309
|
||||
or_test = 310
|
||||
and_test = 311
|
||||
not_test = 312
|
||||
comparison = 313
|
||||
comp_op = 314
|
||||
star_expr = 315
|
||||
expr = 316
|
||||
xor_expr = 317
|
||||
and_expr = 318
|
||||
shift_expr = 319
|
||||
arith_expr = 320
|
||||
term = 321
|
||||
factor = 322
|
||||
power = 323
|
||||
atom_expr = 324
|
||||
atom = 325
|
||||
testlist_comp = 326
|
||||
trailer = 327
|
||||
subscriptlist = 328
|
||||
subscript = 329
|
||||
sliceop = 330
|
||||
exprlist = 331
|
||||
testlist = 332
|
||||
dictorsetmaker = 333
|
||||
classdef = 334
|
||||
arglist = 335
|
||||
argument = 336
|
||||
comp_iter = 337
|
||||
sync_comp_for = 338
|
||||
comp_for = 339
|
||||
comp_if = 340
|
||||
encoding_decl = 341
|
||||
yield_expr = 342
|
||||
yield_arg = 343
|
||||
func_body_suite = 344
|
||||
func_type_input = 345
|
||||
func_type = 346
|
||||
typelist = 347
|
||||
#--end constants--
|
||||
|
||||
sym_name = {}
|
||||
for _name, _value in list(globals().items()):
|
||||
if type(_value) is type(0):
|
||||
sym_name[_value] = _name
|
||||
del _name, _value
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import unittest
|
||||
from test import support
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
import subprocess
|
||||
|
||||
|
||||
SYMBOL_FILE = support.findfile('symbol.py')
|
||||
GEN_SYMBOL_FILE = os.path.join(os.path.dirname(__file__),
|
||||
'..', '..', 'Tools', 'scripts',
|
||||
'generate_symbol_py.py')
|
||||
GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
|
||||
'..', '..', 'Include', 'graminit.h')
|
||||
TEST_PY_FILE = 'symbol_test.py'
|
||||
|
||||
|
||||
class TestSymbolGeneration(unittest.TestCase):
|
||||
|
||||
def _copy_file_without_generated_symbols(self, source_file, dest_file):
|
||||
with open(source_file) as fp:
|
||||
lines = fp.readlines()
|
||||
with open(dest_file, 'w') as fp:
|
||||
fp.writelines(lines[:lines.index("#--start constants--\n") + 1])
|
||||
fp.writelines(lines[lines.index("#--end constants--\n"):])
|
||||
|
||||
def _generate_symbols(self, grammar_file, target_symbol_py_file):
|
||||
proc = subprocess.Popen([sys.executable,
|
||||
GEN_SYMBOL_FILE,
|
||||
grammar_file,
|
||||
target_symbol_py_file], stderr=subprocess.PIPE)
|
||||
stderr = proc.communicate()[1]
|
||||
return proc.returncode, stderr
|
||||
|
||||
def compare_files(self, file1, file2):
|
||||
with open(file1) as fp:
|
||||
lines1 = fp.readlines()
|
||||
with open(file2) as fp:
|
||||
lines2 = fp.readlines()
|
||||
self.assertEqual(lines1, lines2)
|
||||
|
||||
@unittest.skipUnless(sysconfig.is_python_build(),
|
||||
'test only works from source build directory')
|
||||
def test_real_grammar_and_symbol_file(self):
|
||||
output = support.TESTFN
|
||||
self.addCleanup(support.unlink, output)
|
||||
|
||||
self._copy_file_without_generated_symbols(SYMBOL_FILE, output)
|
||||
|
||||
exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output)
|
||||
self.assertEqual(b'', stderr)
|
||||
self.assertEqual(0, exitcode)
|
||||
|
||||
self.compare_files(SYMBOL_FILE, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -344,7 +344,6 @@ PYTHON_OBJS= \
|
|||
Python/getcopyright.o \
|
||||
Python/getplatform.o \
|
||||
Python/getversion.o \
|
||||
Python/graminit.o \
|
||||
Python/hamt.o \
|
||||
Python/hashtable.o \
|
||||
Python/import.o \
|
||||
|
|
@ -742,7 +741,7 @@ regen-importlib: Programs/_freeze_importlib
|
|||
# Regenerate all generated files
|
||||
|
||||
regen-all: regen-opcode regen-opcode-targets regen-typeslots \
|
||||
regen-token regen-symbol regen-ast regen-importlib clinic \
|
||||
regen-token regen-ast regen-importlib clinic \
|
||||
regen-pegen-metaparser regen-pegen
|
||||
|
||||
############################################################################
|
||||
|
|
@ -881,15 +880,7 @@ regen-keyword:
|
|||
$(srcdir)/Lib/keyword.py.new
|
||||
$(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new
|
||||
|
||||
.PHONY: regen-symbol
|
||||
regen-symbol: $(srcdir)/Include/graminit.h
|
||||
# Regenerate Lib/symbol.py from Include/graminit.h
|
||||
# using Tools/scripts/generate_symbol_py.py
|
||||
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_symbol_py.py \
|
||||
$(srcdir)/Include/graminit.h \
|
||||
$(srcdir)/Lib/symbol.py
|
||||
|
||||
Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o Parser/parsetok.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
|
||||
Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/Python-ast.h
|
||||
|
||||
Python/getplatform.o: $(srcdir)/Python/getplatform.c
|
||||
$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
|
||||
|
|
@ -989,7 +980,6 @@ PYTHON_HEADERS= \
|
|||
$(srcdir)/Include/abstract.h \
|
||||
$(srcdir)/Include/asdl.h \
|
||||
$(srcdir)/Include/ast.h \
|
||||
$(srcdir)/Include/bitset.h \
|
||||
$(srcdir)/Include/bltinmodule.h \
|
||||
$(srcdir)/Include/boolobject.h \
|
||||
$(srcdir)/Include/bytearrayobject.h \
|
||||
|
|
@ -1027,7 +1017,6 @@ PYTHON_HEADERS= \
|
|||
$(srcdir)/Include/modsupport.h \
|
||||
$(srcdir)/Include/moduleobject.h \
|
||||
$(srcdir)/Include/namespaceobject.h \
|
||||
$(srcdir)/Include/node.h \
|
||||
$(srcdir)/Include/object.h \
|
||||
$(srcdir)/Include/objimpl.h \
|
||||
$(srcdir)/Include/odictobject.h \
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Remove the remaining files from the old parser and the :mod:`symbol` module.
|
||||
|
|
@ -115,7 +115,6 @@
|
|||
<ClInclude Include="..\Include\abstract.h" />
|
||||
<ClInclude Include="..\Include\asdl.h" />
|
||||
<ClInclude Include="..\Include\ast.h" />
|
||||
<ClInclude Include="..\Include\bitset.h" />
|
||||
<ClInclude Include="..\Include\boolobject.h" />
|
||||
<ClInclude Include="..\Include\bytearrayobject.h" />
|
||||
<ClInclude Include="..\Include\bytesobject.h" />
|
||||
|
|
@ -162,8 +161,6 @@
|
|||
<ClInclude Include="..\Include\frameobject.h" />
|
||||
<ClInclude Include="..\Include\funcobject.h" />
|
||||
<ClInclude Include="..\Include\genobject.h" />
|
||||
<ClInclude Include="..\Include\graminit.h" />
|
||||
<ClInclude Include="..\Include\grammar.h" />
|
||||
<ClInclude Include="..\Include\import.h" />
|
||||
<ClInclude Include="..\Include\internal\pegen_interface.h" />
|
||||
<ClInclude Include="..\Include\internal\pycore_abstract.h" />
|
||||
|
|
@ -209,14 +206,12 @@
|
|||
<ClInclude Include="..\Include\modsupport.h" />
|
||||
<ClInclude Include="..\Include\moduleobject.h" />
|
||||
<ClInclude Include="..\Include\namespaceobject.h" />
|
||||
<ClInclude Include="..\Include\node.h" />
|
||||
<ClInclude Include="..\Include\object.h" />
|
||||
<ClInclude Include="..\Include\objimpl.h" />
|
||||
<ClInclude Include="..\Include\odictobject.h" />
|
||||
<ClInclude Include="..\Include\opcode.h" />
|
||||
<ClInclude Include="..\Include\osdefs.h" />
|
||||
<ClInclude Include="..\Include\osmodule.h" />
|
||||
<ClInclude Include="..\Include\parsetok.h" />
|
||||
<ClInclude Include="..\Include\patchlevel.h" />
|
||||
<ClInclude Include="..\Include\picklebufobject.h" />
|
||||
<ClInclude Include="..\Include\py_curses.h" />
|
||||
|
|
@ -451,7 +446,6 @@
|
|||
<ClCompile Include="..\Python\getopt.c" />
|
||||
<ClCompile Include="..\Python\getplatform.c" />
|
||||
<ClCompile Include="..\Python\getversion.c" />
|
||||
<ClCompile Include="..\Python\graminit.c" />
|
||||
<ClCompile Include="..\Python\hamt.c" />
|
||||
<ClCompile Include="..\Python\hashtable.c" />
|
||||
<ClCompile Include="..\Python\import.c" />
|
||||
|
|
|
|||
|
|
@ -42,9 +42,6 @@
|
|||
<ClInclude Include="..\Include\ast.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\bitset.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\boolobject.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
|
|
@ -183,12 +180,6 @@
|
|||
<ClInclude Include="..\Include\genobject.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\graminit.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\grammar.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\import.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
|
|
@ -318,9 +309,6 @@
|
|||
<ClInclude Include="..\Include\moduleobject.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\node.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\object.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
|
|
@ -336,9 +324,6 @@
|
|||
<ClInclude Include="..\Include\osmodule.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\parsetok.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\Include\patchlevel.h">
|
||||
<Filter>Include</Filter>
|
||||
</ClInclude>
|
||||
|
|
@ -1010,9 +995,6 @@
|
|||
<ClCompile Include="..\Python\getversion.c">
|
||||
<Filter>Python</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Python\graminit.c">
|
||||
<Filter>Python</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\Python\hamt.h">
|
||||
<Filter>Python</Filter>
|
||||
</ClCompile>
|
||||
|
|
|
|||
|
|
@ -133,10 +133,6 @@
|
|||
</None>
|
||||
<None Include="..\Grammar\Tokens">
|
||||
</None>
|
||||
<None Include="..\Include\graminit.h">
|
||||
</None>
|
||||
<None Include="..\Python\graminit.c">
|
||||
</None>
|
||||
<None Include="..\Include\token.h">
|
||||
</None>
|
||||
<None Include="..\Include\opcode.h">
|
||||
|
|
@ -202,18 +198,12 @@
|
|||
</Copy>
|
||||
<Warning Text="Keywords updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
|
||||
</Target>
|
||||
<Target Name="_RegenSymbols" AfterTargets="_RegenKeywords">
|
||||
<!-- Regenerate Lib/symbol.py from Include/graminit.h using Tools/scripts/generate_symbol_py.py-->
|
||||
<Exec Command=""$(PythonExe)" $(PySourcePath)Tools\scripts\generate_symbol_py.py "$(PySourcePath)Include\graminit.h" "$(PySourcePath)Lib\symbol.py"" />
|
||||
</Target>
|
||||
<Target Name="_CleanFiles" BeforeTargets="CoreClean">
|
||||
<ItemGroup>
|
||||
<Clean Include="$(IntDir)keyword.py" />
|
||||
<Clean Include="$(IntDir)opcode.h" />
|
||||
<Clean Include="$(IntDir)Python-ast.c" />
|
||||
<Clean Include="$(IntDir)Python-ast.h" />
|
||||
<Clean Include="$(IntDir)graminit.h.new" />
|
||||
<Clean Include="$(IntDir)graminit.c.new" />
|
||||
</ItemGroup>
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
|
|||
189
Parser/node.c
189
Parser/node.c
|
|
@ -1,189 +0,0 @@
|
|||
/* Parse tree node implementation */
|
||||
|
||||
#include "Python.h"
|
||||
#include "node.h"
|
||||
#include "errcode.h"
|
||||
|
||||
node *
|
||||
PyNode_New(int type)
|
||||
{
|
||||
node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
|
||||
if (n == NULL)
|
||||
return NULL;
|
||||
n->n_type = type;
|
||||
n->n_str = NULL;
|
||||
n->n_lineno = 0;
|
||||
n->n_end_lineno = 0;
|
||||
n->n_col_offset = 0;
|
||||
n->n_end_col_offset = -1;
|
||||
n->n_nchildren = 0;
|
||||
n->n_child = NULL;
|
||||
return n;
|
||||
}
|
||||
|
||||
/* See comments at XXXROUNDUP below. Returns -1 on overflow. */
|
||||
static int
|
||||
fancy_roundup(int n)
|
||||
{
|
||||
/* Round up to the closest power of 2 >= n. */
|
||||
int result = 256;
|
||||
assert(n > 128);
|
||||
while (result < n) {
|
||||
result <<= 1;
|
||||
if (result <= 0)
|
||||
return -1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* A gimmick to make massive numbers of reallocs quicker. The result is
|
||||
* a number >= the input. In PyNode_AddChild, it's used like so, when
|
||||
* we're about to add child number current_size + 1:
|
||||
*
|
||||
* if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1):
|
||||
* allocate space for XXXROUNDUP(current_size + 1) total children
|
||||
* else:
|
||||
* we already have enough space
|
||||
*
|
||||
* Since a node starts out empty, we must have
|
||||
*
|
||||
* XXXROUNDUP(0) < XXXROUNDUP(1)
|
||||
*
|
||||
* so that we allocate space for the first child. One-child nodes are very
|
||||
* common (presumably that would change if we used a more abstract form
|
||||
* of syntax tree), so to avoid wasting memory it's desirable that
|
||||
* XXXROUNDUP(1) == 1. That in turn forces XXXROUNDUP(0) == 0.
|
||||
*
|
||||
* Else for 2 <= n <= 128, we round up to the closest multiple of 4. Why 4?
|
||||
* Rounding up to a multiple of an exact power of 2 is very efficient, and
|
||||
* most nodes with more than one child have <= 4 kids.
|
||||
*
|
||||
* Else we call fancy_roundup() to grow proportionately to n. We've got an
|
||||
* extreme case then (like test_longexp.py), and on many platforms doing
|
||||
* anything less than proportional growth leads to exorbitant runtime
|
||||
* (e.g., MacPython), or extreme fragmentation of user address space (e.g.,
|
||||
* Win98).
|
||||
*
|
||||
* In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
|
||||
* reported that, with this scheme, 89% of PyObject_REALLOC calls in
|
||||
* PyNode_AddChild passed 1 for the size, and 9% passed 4. So this usually
|
||||
* wastes very little memory, but is very effective at sidestepping
|
||||
* platform-realloc disasters on vulnerable platforms.
|
||||
*
|
||||
* Note that this would be straightforward if a node stored its current
|
||||
* capacity. The code is tricky to avoid that.
|
||||
*/
|
||||
#define XXXROUNDUP(n) ((n) <= 1 ? (n) : \
|
||||
(n) <= 128 ? (int)_Py_SIZE_ROUND_UP((n), 4) : \
|
||||
fancy_roundup(n))
|
||||
|
||||
|
||||
void
|
||||
_PyNode_FinalizeEndPos(node *n)
|
||||
{
|
||||
int nch = NCH(n);
|
||||
node *last;
|
||||
if (nch == 0) {
|
||||
return;
|
||||
}
|
||||
last = CHILD(n, nch - 1);
|
||||
_PyNode_FinalizeEndPos(last);
|
||||
n->n_end_lineno = last->n_end_lineno;
|
||||
n->n_end_col_offset = last->n_end_col_offset;
|
||||
}
|
||||
|
||||
int
|
||||
PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset,
|
||||
int end_lineno, int end_col_offset)
|
||||
{
|
||||
const int nch = n1->n_nchildren;
|
||||
int current_capacity;
|
||||
int required_capacity;
|
||||
node *n;
|
||||
|
||||
// finalize end position of previous node (if any)
|
||||
if (nch > 0) {
|
||||
_PyNode_FinalizeEndPos(CHILD(n1, nch - 1));
|
||||
}
|
||||
|
||||
if (nch == INT_MAX || nch < 0)
|
||||
return E_OVERFLOW;
|
||||
|
||||
current_capacity = XXXROUNDUP(nch);
|
||||
required_capacity = XXXROUNDUP(nch + 1);
|
||||
if (current_capacity < 0 || required_capacity < 0)
|
||||
return E_OVERFLOW;
|
||||
if (current_capacity < required_capacity) {
|
||||
if ((size_t)required_capacity > SIZE_MAX / sizeof(node)) {
|
||||
return E_NOMEM;
|
||||
}
|
||||
n = n1->n_child;
|
||||
n = (node *) PyObject_REALLOC(n,
|
||||
required_capacity * sizeof(node));
|
||||
if (n == NULL)
|
||||
return E_NOMEM;
|
||||
n1->n_child = n;
|
||||
}
|
||||
|
||||
n = &n1->n_child[n1->n_nchildren++];
|
||||
n->n_type = type;
|
||||
n->n_str = str;
|
||||
n->n_lineno = lineno;
|
||||
n->n_col_offset = col_offset;
|
||||
n->n_end_lineno = end_lineno; // this and below will be updates after all children are added.
|
||||
n->n_end_col_offset = end_col_offset;
|
||||
n->n_nchildren = 0;
|
||||
n->n_child = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Forward */
|
||||
static void freechildren(node *);
|
||||
static Py_ssize_t sizeofchildren(node *n);
|
||||
|
||||
|
||||
void
|
||||
PyNode_Free(node *n)
|
||||
{
|
||||
if (n != NULL) {
|
||||
freechildren(n);
|
||||
PyObject_FREE(n);
|
||||
}
|
||||
}
|
||||
|
||||
Py_ssize_t
|
||||
_PyNode_SizeOf(node *n)
|
||||
{
|
||||
Py_ssize_t res = 0;
|
||||
|
||||
if (n != NULL)
|
||||
res = sizeof(node) + sizeofchildren(n);
|
||||
return res;
|
||||
}
|
||||
|
||||
static void
|
||||
freechildren(node *n)
|
||||
{
|
||||
int i;
|
||||
for (i = NCH(n); --i >= 0; )
|
||||
freechildren(CHILD(n, i));
|
||||
if (n->n_child != NULL)
|
||||
PyObject_FREE(n->n_child);
|
||||
if (STR(n) != NULL)
|
||||
PyObject_FREE(STR(n));
|
||||
}
|
||||
|
||||
static Py_ssize_t
|
||||
sizeofchildren(node *n)
|
||||
{
|
||||
Py_ssize_t res = 0;
|
||||
int i;
|
||||
for (i = NCH(n); --i >= 0; )
|
||||
res += sizeofchildren(CHILD(n, i));
|
||||
if (n->n_child != NULL)
|
||||
/* allocated size of n->n_child array */
|
||||
res += XXXROUNDUP(NCH(n)) * sizeof(node);
|
||||
if (STR(n) != NULL)
|
||||
res += strlen(STR(n)) + 1;
|
||||
return res;
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@
|
|||
*/
|
||||
#include "Python.h"
|
||||
#include "Python-ast.h"
|
||||
#include "node.h"
|
||||
#include "ast.h"
|
||||
#include "token.h"
|
||||
#include "pythonrun.h"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
#include "Python.h"
|
||||
#include "Python-ast.h"
|
||||
#include "node.h"
|
||||
#include "token.h"
|
||||
#include "graminit.h"
|
||||
#include "code.h"
|
||||
#include "symtable.h"
|
||||
#include "ast.h"
|
||||
|
|
|
|||
2688
Python/graminit.c
2688
Python/graminit.c
File diff suppressed because it is too large
Load Diff
|
|
@ -3,7 +3,6 @@
|
|||
#include "Python.h"
|
||||
|
||||
#include "Python-ast.h"
|
||||
#include "node.h"
|
||||
#include "ast.h"
|
||||
#include "code.h"
|
||||
#include "symtable.h"
|
||||
|
|
|
|||
|
|
@ -20,13 +20,10 @@
|
|||
#include "pycore_pystate.h" // _PyInterpreterState_GET()
|
||||
#include "pycore_sysmodule.h" // _PySys_Audit()
|
||||
|
||||
#include "node.h" // node
|
||||
#include "token.h" // INDENT
|
||||
#include "parsetok.h" // perrdetail
|
||||
#include "errcode.h" // E_EOF
|
||||
#include "code.h" // PyCodeObject
|
||||
#include "symtable.h" // PySymtable_BuildObject()
|
||||
#include "ast.h" // PyAST_FromNodeObject()
|
||||
#include "marshal.h" // PyMarshal_ReadLongFromFile()
|
||||
|
||||
#include "pegen_interface.h" // PyPegen_ASTFrom*
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
#! /usr/bin/env python3
|
||||
# This script generates the symbol.py source file.
|
||||
|
||||
import sys
|
||||
import re
|
||||
|
||||
def main(inFileName="Include/graminit.h", outFileName="Lib/symbol.py"):
|
||||
try:
|
||||
fp = open(inFileName)
|
||||
except OSError as err:
|
||||
sys.stderr.write("I/O error: %s\n" % str(err))
|
||||
sys.exit(1)
|
||||
with fp:
|
||||
lines = fp.read().split("\n")
|
||||
prog = re.compile(
|
||||
"#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
|
||||
re.IGNORECASE)
|
||||
tokens = {}
|
||||
for line in lines:
|
||||
match = prog.match(line)
|
||||
if match:
|
||||
name, val = match.group(1, 2)
|
||||
val = int(val)
|
||||
tokens[val] = name # reverse so we can sort them...
|
||||
keys = sorted(tokens.keys())
|
||||
# load the output skeleton from the target:
|
||||
try:
|
||||
fp = open(outFileName)
|
||||
except OSError as err:
|
||||
sys.stderr.write("I/O error: %s\n" % str(err))
|
||||
sys.exit(2)
|
||||
with fp:
|
||||
format = fp.read().split("\n")
|
||||
try:
|
||||
start = format.index("#--start constants--") + 1
|
||||
end = format.index("#--end constants--")
|
||||
except ValueError:
|
||||
sys.stderr.write("target does not contain format markers")
|
||||
sys.exit(3)
|
||||
lines = []
|
||||
for val in keys:
|
||||
lines.append("%s = %d" % (tokens[val], val))
|
||||
format[start:end] = lines
|
||||
try:
|
||||
fp = open(outFileName, 'w')
|
||||
except OSError as err:
|
||||
sys.stderr.write("I/O error: %s\n" % str(err))
|
||||
sys.exit(4)
|
||||
with fp:
|
||||
fp.write("\n".join(format))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(*sys.argv[1:])
|
||||
|
|
@ -2739,7 +2739,7 @@ if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then
|
|||
# If we're building out-of-tree, we need to make sure the following
|
||||
# resources get picked up before their $srcdir counterparts.
|
||||
# Objects/ -> typeslots.inc
|
||||
# Include/ -> Python-ast.h, graminit.h
|
||||
# Include/ -> Python-ast.h
|
||||
# Python/ -> importlib.h
|
||||
# (A side effect of this is that these resources will automatically be
|
||||
# regenerated when building out-of-tree, regardless of whether or not
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then
|
|||
# If we're building out-of-tree, we need to make sure the following
|
||||
# resources get picked up before their $srcdir counterparts.
|
||||
# Objects/ -> typeslots.inc
|
||||
# Include/ -> Python-ast.h, graminit.h
|
||||
# Include/ -> Python-ast.h
|
||||
# Python/ -> importlib.h
|
||||
# (A side effect of this is that these resources will automatically be
|
||||
# regenerated when building out-of-tree, regardless of whether or not
|
||||
|
|
|
|||
Loading…
Reference in New Issue