Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-94360: Fix a tokenizer crash when reading encoded files with syntax errors from stdin (#94386) 

* gh-94360: Fix a tokenizer crash when reading encoded files with syntax errors from stdin

Signed-off-by: Pablo Galindo <pablogsal@gmail.com>

* nitty nit

Co-authored-by: Łukasz Langa <lukasz@langa.pl>
This commit is contained in:
Pablo Galindo Salgado 2022-07-05 17:39:21 +01:00 committed by GitHub
parent c8556bcf6c
commit 36fcde61ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS.d/next/Core and Builtins/2022-06-28-14-20-36.gh-issue-94360.DiEnen.rst Normal file
View File

@ -0,0 +1,2 @@
Fixed a tokenizer crash when reading encoded files with syntax errors from
``stdin`` with non utf-8 encoded text. Patch by Pablo Galindo

8
Parser/pegen_errors.c
View File

@ -259,15 +259,15 @@ get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
for (int i = 0; i < relative_lineno - 1; i++) {
char *new_line = strchr(cur_line, '\n') + 1;
char *new_line = strchr(cur_line, '\n');
// The assert is here for debug builds but the conditional that
// follows is there so in release builds we do not crash at the cost
// to report a potentially wrong line.
assert(new_line != NULL && new_line <= buf_end);
if (new_line == NULL || new_line > buf_end) {
assert(new_line != NULL && new_line + 1 < buf_end);
if (new_line == NULL || new_line + 1 > buf_end) {
break;
}
cur_line = new_line;
cur_line = new_line + 1;
}
char *next_newline;

10
Parser/tokenizer.c
View File

@ -308,6 +308,10 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
Py_ssize_t current_size = tok->interactive_src_end - tok->interactive_src_start;
Py_ssize_t line_size = strlen(line);
char last_char = line[line_size > 0 ? line_size - 1 : line_size];
if (last_char != '\n') {
line_size += 1;
}
char* new_str = tok->interactive_src_start;
new_str = PyMem_Realloc(new_str, current_size + line_size + 1);
@ -321,7 +325,11 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
return -1;
}
strcpy(new_str + current_size, line);
if (last_char != '\n') {
/* Last line does not end in \n, fake one */
new_str[current_size + line_size - 1] = '\n';
new_str[current_size + line_size] = '\0';
}
tok->interactive_src_start = new_str;
tok->interactive_src_end = new_str + current_size + line_size;
return 0;