Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-45434: Mark the PyTokenizer C API as private (GH-28924) 

Rename PyTokenize functions to mark them as private:

* PyTokenizer_FindEncodingFilename() => _PyTokenizer_FindEncodingFilename()
* PyTokenizer_FromString() => _PyTokenizer_FromString()
* PyTokenizer_FromFile() => _PyTokenizer_FromFile()
* PyTokenizer_FromUTF8() => _PyTokenizer_FromUTF8()
* PyTokenizer_Free() => _PyTokenizer_Free()
* PyTokenizer_Get() => _PyTokenizer_Get()

Remove the unused PyTokenizer_FindEncoding() function.

import.c: remove unused #include "errcode.h".
This commit is contained in:
Victor Stinner 2021-10-13 17:22:14 +02:00 committed by GitHub
parent 3901c08114
commit 713bb19356
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 40 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Parser/pegen.c
View File

@ -729,7 +729,7 @@ _PyPegen_fill_token(Parser *p)
{
const char *start;
const char *end;
int type = PyTokenizer_Get(p->tok, &start, &end);
int type = _PyTokenizer_Get(p->tok, &start, &end);
// Record and skip '# type: ignore' comments
while (type == TYPE_IGNORE) {
@ -746,7 +746,7 @@ _PyPegen_fill_token(Parser *p)
PyErr_NoMemory();
return -1;
}
type = PyTokenizer_Get(p->tok, &start, &end);
type = _PyTokenizer_Get(p->tok, &start, &end);
}
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
@ -1306,7 +1306,7 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
for (;;) {
const char *start;
const char *end;
switch (PyTokenizer_Get(p->tok, &start, &end)) {
switch (_PyTokenizer_Get(p->tok, &start, &end)) {
case ERRORTOKEN:
if (p->tok->level != 0) {
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
@ -1411,7 +1411,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
const char *enc, const char *ps1, const char *ps2,
PyCompilerFlags *flags, int *errcode, PyArena *arena)
{
struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
if (tok == NULL) {
if (PyErr_Occurred()) {
raise_tokenizer_init_error(filename_ob);
@ -1441,7 +1441,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
_PyPegen_Parser_Free(p);
error:
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return result;
}
@ -1453,9 +1453,9 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
struct tok_state *tok;
if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
tok = PyTokenizer_FromUTF8(str, exec_input);
tok = _PyTokenizer_FromUTF8(str, exec_input);
} else {
tok = PyTokenizer_FromString(str, exec_input);
tok = _PyTokenizer_FromString(str, exec_input);
}
if (tok == NULL) {
if (PyErr_Occurred()) {
@ -1483,7 +1483,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
_PyPegen_Parser_Free(p);
error:
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return result;
}

4
Parser/string_parser.c
View File

@ -386,7 +386,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
str[0] = '(';
str[len+1] = ')';
struct tok_state* tok = PyTokenizer_FromString(str, 1);
struct tok_state* tok = _PyTokenizer_FromString(str, 1);
if (tok == NULL) {
PyMem_Free(str);
return NULL;
@ -409,7 +409,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
exit:
PyMem_Free(str);
_PyPegen_Parser_Free(p2);
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return result;
}

47
Parser/tokenizer.c
View File

@ -108,7 +108,7 @@ static char *
error_ret(struct tok_state *tok) /* XXX */
{
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
if (tok->fp != NULL && tok->buf != NULL) /* see _PyTokenizer_Free */
PyMem_Free(tok->buf);
tok->buf = tok->cur = tok->inp = NULL;
tok->start = NULL;
@ -702,7 +702,7 @@ decode_str(const char *input, int single, struct tok_state *tok)
/* Set up tokenizer for string */
struct tok_state *
PyTokenizer_FromString(const char *str, int exec_input)
_PyTokenizer_FromString(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
char *decoded;
@ -711,7 +711,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
return NULL;
decoded = decode_str(str, exec_input, tok);
if (decoded == NULL) {
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return NULL;
}
@ -723,7 +723,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
/* Set up tokenizer for UTF-8 string */
struct tok_state *
PyTokenizer_FromUTF8(const char *str, int exec_input)
_PyTokenizer_FromUTF8(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
char *translated;
@ -731,7 +731,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
return NULL;
tok->input = translated = translate_newlines(str, exec_input, tok);
if (translated == NULL) {
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return NULL;
}
tok->decoding_state = STATE_NORMAL;
@ -739,7 +739,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
tok->str = translated;
tok->encoding = new_string("utf-8", 5, tok);
if (!tok->encoding) {
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return NULL;
}
@ -751,14 +751,14 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
/* Set up tokenizer for file */
struct tok_state *
PyTokenizer_FromFile(FILE *fp, const char* enc,
const char *ps1, const char *ps2)
_PyTokenizer_FromFile(FILE *fp, const char* enc,
const char *ps1, const char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return NULL;
}
tok->cur = tok->inp = tok->buf;
@ -771,7 +771,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
gets copied into the parse tree. */
tok->encoding = new_string(enc, strlen(enc), tok);
if (!tok->encoding) {
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return NULL;
}
tok->decoding_state = STATE_NORMAL;
@ -782,7 +782,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
/* Free a tok_state structure */
void
PyTokenizer_Free(struct tok_state *tok)
_PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL) {
PyMem_Free(tok->encoding);
@ -2049,7 +2049,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
}
int
PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
_PyTokenizer_Get(struct tok_state *tok,
const char **p_start, const char **p_end)
{
int result = tok_get(tok, p_start, p_end);
if (tok->decoding_erred) {
@ -2062,7 +2063,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Get the encoding of a Python file. Check for the coding cookie and check if
the file starts with a BOM.
PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
_PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
encoding in the first or second line of the file (in which case the encoding
should be assumed to be UTF-8).
@ -2070,7 +2071,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
by the caller. */
char *
PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
{
struct tok_state *tok;
FILE *fp;
@ -2087,7 +2088,7 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
if (fp == NULL) {
return NULL;
}
tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
if (tok == NULL) {
fclose(fp);
return NULL;
@ -2100,12 +2101,12 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
tok->filename = PyUnicode_FromString("<string>");
if (tok->filename == NULL) {
fclose(fp);
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return encoding;
}
}
while (tok->lineno < 2 && tok->done == E_OK) {
PyTokenizer_Get(tok, &p_start, &p_end);
_PyTokenizer_Get(tok, &p_start, &p_end);
}
fclose(fp);
if (tok->encoding) {
@ -2114,18 +2115,11 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
strcpy(encoding, tok->encoding);
}
}
PyTokenizer_Free(tok);
_PyTokenizer_Free(tok);
return encoding;
}
char *
PyTokenizer_FindEncoding(int fd)
{
return PyTokenizer_FindEncodingFilename(fd, NULL);
}
#ifdef Py_DEBUG
void
tok_dump(int type, char *start, char *end)
{
@ -2133,5 +2127,4 @@ tok_dump(int type, char *start, char *end)
if (type == NAME || type == NUMBER || type == STRING || type == OP)
printf("(%.*s)", (int)(end - start), start);
}
#endif
#endif // Py_DEBUG

10
Parser/tokenizer.h
View File

@ -86,12 +86,12 @@ struct tok_state {
enum interactive_underflow_t interactive_underflow;
};
extern struct tok_state *PyTokenizer_FromString(const char *, int);
extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
extern struct tok_state *_PyTokenizer_FromString(const char *, int);
extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int);
extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
const char *, const char *);
extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, const char **, const char **);
extern void _PyTokenizer_Free(struct tok_state *);
extern int _PyTokenizer_Get(struct tok_state *, const char **, const char **);
#define tok_dump _Py_tok_dump

6
Python/Python-tokenize.c
View File

@ -47,7 +47,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
if (filename == NULL) {
return NULL;
}
self->tok = PyTokenizer_FromUTF8(source, 1);
self->tok = _PyTokenizer_FromUTF8(source, 1);
if (self->tok == NULL) {
Py_DECREF(filename);
return NULL;
@ -61,7 +61,7 @@ tokenizeriter_next(tokenizeriterobject *it)
{
const char *start;
const char *end;
int type = PyTokenizer_Get(it->tok, &start, &end);
int type = _PyTokenizer_Get(it->tok, &start, &end);
if (type == ERRORTOKEN && PyErr_Occurred()) {
return NULL;
}
@ -105,7 +105,7 @@ static void
tokenizeriter_dealloc(tokenizeriterobject *it)
{
PyTypeObject *tp = Py_TYPE(it);
PyTokenizer_Free(it->tok);
_PyTokenizer_Free(it->tok);
tp->tp_free(it);
Py_DECREF(tp);
}

1
Python/import.c
View File

@ -11,7 +11,6 @@
#include "pycore_interp.h" // _PyInterpreterState_ClearModules()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_sysmodule.h"
#include "errcode.h"
#include "marshal.h"
#include "code.h"
#include "importdl.h"

4
Python/traceback.c
View File

@ -29,7 +29,7 @@
#define MAX_NTHREADS 100
/* Function from Parser/tokenizer.c */
extern char * PyTokenizer_FindEncodingFilename(int, PyObject *);
extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *);
_Py_IDENTIFIER(TextIOWrapper);
_Py_IDENTIFIER(close);
@ -431,7 +431,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i
Py_DECREF(binary);
return 0;
}
found_encoding = PyTokenizer_FindEncodingFilename(fd, filename);
found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename);
if (found_encoding == NULL)
PyErr_Clear();
encoding = (found_encoding != NULL) ? found_encoding : "utf-8";