From da5727a120e426ffaf68bf3a8016491205bd2f80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 11 May 2022 23:22:50 +0200 Subject: [PATCH] gh-92651: Remove the Include/token.h header file (#92652) Remove the token.h header file. There was never any public tokenizer C API. The token.h header file was only designed to be used by Python internals. Move Include/token.h to Include/internal/pycore_token.h. Including this header file now requires that the Py_BUILD_CORE macro is defined. It no longer checks for the Py_LIMITED_API macro. Rename functions: * PyToken_OneChar() => _PyToken_OneChar() * PyToken_TwoChars() => _PyToken_TwoChars() * PyToken_ThreeChars() => _PyToken_ThreeChars() --- Doc/whatsnew/3.12.rst | 5 ++++ Include/{token.h => internal/pycore_token.h} | 19 +++++++----- Makefile.pre.in | 6 ++-- ...2-05-11-02-33-10.gh-issue-92651.FIXLf0.rst | 3 ++ PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 6 ++-- PCbuild/regen.targets | 2 +- Parser/pegen.h | 2 +- Parser/token.c | 8 ++--- Parser/tokenizer.c | 6 ++-- Parser/tokenizer.h | 2 +- Python/pythonrun.c | 1 - Tools/scripts/generate_token.py | 29 ++++++++++--------- 13 files changed, 52 insertions(+), 39 deletions(-) rename Include/{token.h => internal/pycore_token.h} (87%) create mode 100644 Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 461d9db7934..bc354c36cda 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -139,3 +139,8 @@ Deprecated Removed ------- + +* Remove the ``token.h`` header file. There was never any public tokenizer C + API. The ``token.h`` header file was only designed to be used by Python + internals. + (Contributed by Victor Stinner in :gh:`92651`.) diff --git a/Include/token.h b/Include/internal/pycore_token.h similarity index 87% rename from Include/token.h rename to Include/internal/pycore_token.h index eb1b9ea47b4..f9b8240e216 100644 --- a/Include/token.h +++ b/Include/internal/pycore_token.h @@ -1,13 +1,16 @@ /* Auto-generated by Tools/scripts/generate_token.py */ /* Token types */ -#ifndef Py_LIMITED_API -#ifndef Py_TOKEN_H -#define Py_TOKEN_H +#ifndef Py_INTERNAL_TOKEN_H +#define Py_INTERNAL_TOKEN_H #ifdef __cplusplus extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ #define ENDMARKER 0 @@ -85,13 +88,13 @@ extern "C" { (x) == DEDENT) +// Symbols exported for test_peg_generator PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ -PyAPI_FUNC(int) PyToken_OneChar(int); -PyAPI_FUNC(int) PyToken_TwoChars(int, int); -PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); +PyAPI_FUNC(int) _PyToken_OneChar(int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); #ifdef __cplusplus } #endif -#endif /* !Py_TOKEN_H */ -#endif /* Py_LIMITED_API */ +#endif // !Py_INTERNAL_TOKEN_H diff --git a/Makefile.pre.in b/Makefile.pre.in index e45d4fe3ecb..869c78ee0d3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1325,11 +1325,11 @@ regen-token: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Doc/library/token-list.inc - # Regenerate Include/token.h from Grammar/Tokens + # Regenerate Include/internal/pycore_token.h from Grammar/Tokens # using Tools/scripts/generate_token.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \ $(srcdir)/Grammar/Tokens \ - $(srcdir)/Include/token.h + $(srcdir)/Include/internal/pycore_token.h # Regenerate Parser/token.c from Grammar/Tokens # using Tools/scripts/generate_token.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \ @@ -1521,7 +1521,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/structmember.h \ $(srcdir)/Include/structseq.h \ $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/token.h \ $(srcdir)/Include/traceback.h \ $(srcdir)/Include/tracemalloc.h \ $(srcdir)/Include/tupleobject.h \ @@ -1632,6 +1631,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ diff --git a/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst b/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst new file mode 100644 index 00000000000..60a8818e46b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst @@ -0,0 +1,3 @@ +Remove the ``token.h`` header file. There was never any public tokenizer C +API. The ``token.h`` header file was only designed to be used by Python +internals. Patch by Victor Stinner. diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3ce116d2bab..a35884b3c35 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -244,6 +244,7 @@ + @@ -291,7 +292,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 542d5510456..ff42cc92c4b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -213,9 +213,6 @@ Include - - Include - Include @@ -633,6 +630,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 24b5ced1de0..9073bb6ab2b 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -19,7 +19,7 @@ <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst - <_TokenOutputs Include="$(PySourcePath)Include\token.h"> + <_TokenOutputs Include="$(PySourcePath)Include\internal\pycore_token.h"> h <_TokenOutputs Include="$(PySourcePath)Parser\token.c"> diff --git a/Parser/pegen.h b/Parser/pegen.h index fe0c327b875..d6a6e4e1eeb 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -3,8 +3,8 @@ #define PY_SSIZE_T_CLEAN #include -#include #include +#include #if 0 #define PyPARSE_YIELD_IS_KEYWORD 0x0001 diff --git a/Parser/token.c b/Parser/token.c index 74bca0eff65..fa03fbc450b 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -1,7 +1,7 @@ /* Auto-generated by Tools/scripts/generate_token.py */ #include "Python.h" -#include "token.h" +#include "pycore_token.h" /* Token names */ @@ -76,7 +76,7 @@ const char * const _PyParser_TokenNames[] = { /* Return the token corresponding to a single character */ int -PyToken_OneChar(int c1) +_PyToken_OneChar(int c1) { switch (c1) { case '%': return PERCENT; @@ -107,7 +107,7 @@ PyToken_OneChar(int c1) } int -PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2) { switch (c1) { case '!': @@ -191,7 +191,7 @@ PyToken_TwoChars(int c1, int c2) } int -PyToken_ThreeChars(int c1, int c2, int c3) +_PyToken_ThreeChars(int c1, int c2, int c3) { switch (c1) { case '*': diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index c450aa8e463..7c797180956 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Check for two-character token */ { int c2 = tok_nextc(tok); - int token = PyToken_TwoChars(c, c2); + int token = _PyToken_TwoChars(c, c2); if (token != OP) { int c3 = tok_nextc(tok); - int token3 = PyToken_ThreeChars(c, c2, c3); + int token3 = _PyToken_ThreeChars(c, c2, c3); if (token3 != OP) { token = token3; } @@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Punctuation character */ *p_start = tok->start; *p_end = tok->cur; - return PyToken_OneChar(c); + return _PyToken_OneChar(c); } int diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0cb665104b2..dba71bd60fe 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -8,7 +8,7 @@ extern "C" { /* Tokenizer interface */ -#include "token.h" /* For token types */ +#include "pycore_token.h" /* For token types */ #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f12b9f6e953..202df585f31 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -24,7 +24,6 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() -#include "token.h" // INDENT #include "errcode.h" // E_EOF #include "marshal.h" // PyMarshal_ReadLongFromFile() diff --git a/Tools/scripts/generate_token.py b/Tools/scripts/generate_token.py index 77bb5bd5eca..d8be8b93de1 100755 --- a/Tools/scripts/generate_token.py +++ b/Tools/scripts/generate_token.py @@ -51,13 +51,16 @@ token_h_template = """\ /* Auto-generated by Tools/scripts/generate_token.py */ /* Token types */ -#ifndef Py_LIMITED_API -#ifndef Py_TOKEN_H -#define Py_TOKEN_H +#ifndef Py_INTERNAL_TOKEN_H +#define Py_INTERNAL_TOKEN_H #ifdef __cplusplus extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ %s\ @@ -75,19 +78,19 @@ extern "C" { (x) == DEDENT) +// Symbols exported for test_peg_generator PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ -PyAPI_FUNC(int) PyToken_OneChar(int); -PyAPI_FUNC(int) PyToken_TwoChars(int, int); -PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); +PyAPI_FUNC(int) _PyToken_OneChar(int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); #ifdef __cplusplus } #endif -#endif /* !Py_TOKEN_H */ -#endif /* Py_LIMITED_API */ +#endif // !Py_INTERNAL_TOKEN_H """ -def make_h(infile, outfile='Include/token.h'): +def make_h(infile, outfile='Include/internal/pycore_token.h'): tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile) defines = [] @@ -106,7 +109,7 @@ token_c_template = """\ /* Auto-generated by Tools/scripts/generate_token.py */ #include "Python.h" -#include "token.h" +#include "pycore_token.h" /* Token names */ @@ -117,21 +120,21 @@ const char * const _PyParser_TokenNames[] = { /* Return the token corresponding to a single character */ int -PyToken_OneChar(int c1) +_PyToken_OneChar(int c1) { %s\ return OP; } int -PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2) { %s\ return OP; } int -PyToken_ThreeChars(int c1, int c2, int c3) +_PyToken_ThreeChars(int c1, int c2, int c3) { %s\ return OP;