diff --git a/Grammar/python.gram b/Grammar/python.gram index 40ca3dc8d12..0ff2dcca884 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -323,7 +323,8 @@ compare_op_bitwise_or_pair[CmpopExprPair*]: | isnot_bitwise_or | is_bitwise_or eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) } -noteq_bitwise_or[CmpopExprPair*]: '!=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotEq, a) } +noteq_bitwise_or[CmpopExprPair*]: + | (tok='!=' {_PyPegen_check_barry_as_flufl(p) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } diff --git a/Include/internal/pegen_interface.h b/Include/internal/pegen_interface.h index d8621c1a889..adff7315681 100644 --- a/Include/internal/pegen_interface.h +++ b/Include/internal/pegen_interface.h @@ -11,21 +11,24 @@ extern "C" { #include "Python.h" #include "Python-ast.h" -PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyCompilerFlags*, PyArena *arena); PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, int mode, PyCompilerFlags *flags, PyArena *arena); PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, int mode, const char *enc, const char *ps1, - const char *ps2, int *errcode, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode); + const char *ps2, PyCompilerFlags *flags, + int *errcode, PyArena *arena); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode, PyCompilerFlags *flags); PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, int mode, PyCompilerFlags *flags); PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFileObject(FILE *, PyObject *filename_ob, - int mode, const char *enc, + int mode, const char *ps1, const char *ps2, + PyCompilerFlags *flags, + const char *enc, int *errcode); #ifdef __cplusplus diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index 35ab934ab37..b71442804c7 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -4,7 +4,6 @@ import sys from test import support -@support.skip_if_new_parser("Not supported by pegen yet") class FLUFLTests(unittest.TestCase): def test_barry_as_bdfl(self): @@ -16,10 +15,13 @@ class FLUFLTests(unittest.TestCase): __future__.CO_FUTURE_BARRY_AS_BDFL) self.assertRegex(str(cm.exception), "with Barry as BDFL, use '<>' instead of '!='") - self.assertEqual(cm.exception.text, '2 != 3\n') + self.assertIn('2 != 3', cm.exception.text) self.assertEqual(cm.exception.filename, '') - self.assertEqual(cm.exception.lineno, 2) - self.assertEqual(cm.exception.offset, 4) + + self.assertTrue(cm.exception.lineno, 2) + # The old parser reports the end of the token and the new + # parser reports the start of the token + self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3) def test_guido_as_bdfl(self): code = '2 {0} 3' @@ -27,10 +29,12 @@ class FLUFLTests(unittest.TestCase): with self.assertRaises(SyntaxError) as cm: compile(code.format('<>'), '', 'exec') self.assertRegex(str(cm.exception), "invalid syntax") - self.assertEqual(cm.exception.text, '2 <> 3\n') + self.assertIn('2 <> 3', cm.exception.text) self.assertEqual(cm.exception.filename, '') self.assertEqual(cm.exception.lineno, 1) - self.assertEqual(cm.exception.offset, 4) + # The old parser reports the end of the token and the new + # parser reports the start of the token + self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3) if __name__ == '__main__': diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c index cb5f9aa63ae..e1ec36e07bd 100644 --- a/Modules/_peg_parser.c +++ b/Modules/_peg_parser.c @@ -28,9 +28,10 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + PyCompilerFlags flags = _PyCompilerFlags_INIT; PyObject *result = NULL; - mod_ty res = PyPegen_ASTFromFile(filename, mode, arena); + mod_ty res = PyPegen_ASTFromFile(filename, mode, &flags, arena); if (res == NULL) { goto error; } diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c index 25607eaf73c..b26f7327bd2 100644 --- a/Parser/pegen/parse.c +++ b/Parser/pegen/parse.c @@ -297,37 +297,37 @@ static KeywordToken *reserved_keywords[] = { #define _loop1_83_type 1226 #define _loop1_84_type 1227 #define _loop1_85_type 1228 -#define _loop0_87_type 1229 -#define _gather_86_type 1230 -#define _tmp_88_type 1231 +#define _tmp_86_type 1229 +#define _loop0_88_type 1230 +#define _gather_87_type 1231 #define _tmp_89_type 1232 #define _tmp_90_type 1233 #define _tmp_91_type 1234 -#define _loop1_92_type 1235 -#define _tmp_93_type 1236 +#define _tmp_92_type 1235 +#define _loop1_93_type 1236 #define _tmp_94_type 1237 -#define _loop0_96_type 1238 -#define _gather_95_type 1239 -#define _loop1_97_type 1240 -#define _tmp_98_type 1241 +#define _tmp_95_type 1238 +#define _loop0_97_type 1239 +#define _gather_96_type 1240 +#define _loop1_98_type 1241 #define _tmp_99_type 1242 -#define _loop0_101_type 1243 -#define _gather_100_type 1244 -#define _loop0_103_type 1245 -#define _gather_102_type 1246 -#define _loop0_105_type 1247 -#define _gather_104_type 1248 -#define _loop0_107_type 1249 -#define _gather_106_type 1250 -#define _loop0_108_type 1251 -#define _loop0_110_type 1252 -#define _gather_109_type 1253 -#define _tmp_111_type 1254 -#define _loop0_113_type 1255 -#define _gather_112_type 1256 -#define _loop0_115_type 1257 -#define _gather_114_type 1258 -#define _tmp_116_type 1259 +#define _tmp_100_type 1243 +#define _loop0_102_type 1244 +#define _gather_101_type 1245 +#define _loop0_104_type 1246 +#define _gather_103_type 1247 +#define _loop0_106_type 1248 +#define _gather_105_type 1249 +#define _loop0_108_type 1250 +#define _gather_107_type 1251 +#define _loop0_109_type 1252 +#define _loop0_111_type 1253 +#define _gather_110_type 1254 +#define _tmp_112_type 1255 +#define _loop0_114_type 1256 +#define _gather_113_type 1257 +#define _loop0_116_type 1258 +#define _gather_115_type 1259 #define _tmp_117_type 1260 #define _tmp_118_type 1261 #define _tmp_119_type 1262 @@ -346,8 +346,9 @@ static KeywordToken *reserved_keywords[] = { #define _tmp_132_type 1275 #define _tmp_133_type 1276 #define _tmp_134_type 1277 -#define _loop0_135_type 1278 -#define _tmp_136_type 1279 +#define _tmp_135_type 1278 +#define _loop0_136_type 1279 +#define _tmp_137_type 1280 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -578,37 +579,37 @@ static asdl_seq *_gather_81_rule(Parser *p); static asdl_seq *_loop1_83_rule(Parser *p); static asdl_seq *_loop1_84_rule(Parser *p); static asdl_seq *_loop1_85_rule(Parser *p); -static asdl_seq *_loop0_87_rule(Parser *p); -static asdl_seq *_gather_86_rule(Parser *p); -static void *_tmp_88_rule(Parser *p); +static void *_tmp_86_rule(Parser *p); +static asdl_seq *_loop0_88_rule(Parser *p); +static asdl_seq *_gather_87_rule(Parser *p); static void *_tmp_89_rule(Parser *p); static void *_tmp_90_rule(Parser *p); static void *_tmp_91_rule(Parser *p); -static asdl_seq *_loop1_92_rule(Parser *p); -static void *_tmp_93_rule(Parser *p); +static void *_tmp_92_rule(Parser *p); +static asdl_seq *_loop1_93_rule(Parser *p); static void *_tmp_94_rule(Parser *p); -static asdl_seq *_loop0_96_rule(Parser *p); -static asdl_seq *_gather_95_rule(Parser *p); -static asdl_seq *_loop1_97_rule(Parser *p); -static void *_tmp_98_rule(Parser *p); +static void *_tmp_95_rule(Parser *p); +static asdl_seq *_loop0_97_rule(Parser *p); +static asdl_seq *_gather_96_rule(Parser *p); +static asdl_seq *_loop1_98_rule(Parser *p); static void *_tmp_99_rule(Parser *p); -static asdl_seq *_loop0_101_rule(Parser *p); -static asdl_seq *_gather_100_rule(Parser *p); -static asdl_seq *_loop0_103_rule(Parser *p); -static asdl_seq *_gather_102_rule(Parser *p); -static asdl_seq *_loop0_105_rule(Parser *p); -static asdl_seq *_gather_104_rule(Parser *p); -static asdl_seq *_loop0_107_rule(Parser *p); -static asdl_seq *_gather_106_rule(Parser *p); +static void *_tmp_100_rule(Parser *p); +static asdl_seq *_loop0_102_rule(Parser *p); +static asdl_seq *_gather_101_rule(Parser *p); +static asdl_seq *_loop0_104_rule(Parser *p); +static asdl_seq *_gather_103_rule(Parser *p); +static asdl_seq *_loop0_106_rule(Parser *p); +static asdl_seq *_gather_105_rule(Parser *p); static asdl_seq *_loop0_108_rule(Parser *p); -static asdl_seq *_loop0_110_rule(Parser *p); -static asdl_seq *_gather_109_rule(Parser *p); -static void *_tmp_111_rule(Parser *p); -static asdl_seq *_loop0_113_rule(Parser *p); -static asdl_seq *_gather_112_rule(Parser *p); -static asdl_seq *_loop0_115_rule(Parser *p); -static asdl_seq *_gather_114_rule(Parser *p); -static void *_tmp_116_rule(Parser *p); +static asdl_seq *_gather_107_rule(Parser *p); +static asdl_seq *_loop0_109_rule(Parser *p); +static asdl_seq *_loop0_111_rule(Parser *p); +static asdl_seq *_gather_110_rule(Parser *p); +static void *_tmp_112_rule(Parser *p); +static asdl_seq *_loop0_114_rule(Parser *p); +static asdl_seq *_gather_113_rule(Parser *p); +static asdl_seq *_loop0_116_rule(Parser *p); +static asdl_seq *_gather_115_rule(Parser *p); static void *_tmp_117_rule(Parser *p); static void *_tmp_118_rule(Parser *p); static void *_tmp_119_rule(Parser *p); @@ -627,8 +628,9 @@ static void *_tmp_131_rule(Parser *p); static void *_tmp_132_rule(Parser *p); static void *_tmp_133_rule(Parser *p); static void *_tmp_134_rule(Parser *p); -static asdl_seq *_loop0_135_rule(Parser *p); -static void *_tmp_136_rule(Parser *p); +static void *_tmp_135_rule(Parser *p); +static asdl_seq *_loop0_136_rule(Parser *p); +static void *_tmp_137_rule(Parser *p); // file: statements? $ @@ -5557,7 +5559,7 @@ eq_bitwise_or_rule(Parser *p) return res; } -// noteq_bitwise_or: '!=' bitwise_or +// noteq_bitwise_or: ('!=') bitwise_or static CmpopExprPair* noteq_bitwise_or_rule(Parser *p) { @@ -5566,11 +5568,11 @@ noteq_bitwise_or_rule(Parser *p) } CmpopExprPair* res = NULL; int mark = p->mark; - { // '!=' bitwise_or + { // ('!=') bitwise_or + void *_tmp_86_var; expr_ty a; - void *literal; if ( - (literal = _PyPegen_expect_token(p, 28)) + (_tmp_86_var = _tmp_86_rule(p)) && (a = bitwise_or_rule(p)) ) @@ -7012,7 +7014,7 @@ slices_rule(Parser *p) void *opt_var; UNUSED(opt_var); // Silence compiler warnings if ( - (a = _gather_86_rule(p)) + (a = _gather_87_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -7068,7 +7070,7 @@ slice_rule(Parser *p) && (b = expression_rule(p), 1) && - (c = _tmp_88_rule(p), 1) + (c = _tmp_89_rule(p), 1) ) { Token *token = _PyPegen_get_last_nonnwhitespace_token(p); @@ -7256,22 +7258,9 @@ atom_rule(Parser *p) p->mark = mark; } { // &'(' (tuple | group | genexp) - void *_tmp_89_var; - if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) - && - (_tmp_89_var = _tmp_89_rule(p)) - ) - { - res = _tmp_89_var; - goto done; - } - p->mark = mark; - } - { // &'[' (list | listcomp) void *_tmp_90_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) && (_tmp_90_var = _tmp_90_rule(p)) ) @@ -7281,10 +7270,10 @@ atom_rule(Parser *p) } p->mark = mark; } - { // &'{' (dict | set | dictcomp | setcomp) + { // &'[' (list | listcomp) void *_tmp_91_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) && (_tmp_91_var = _tmp_91_rule(p)) ) @@ -7294,6 +7283,19 @@ atom_rule(Parser *p) } p->mark = mark; } + { // &'{' (dict | set | dictcomp | setcomp) + void *_tmp_92_var; + if ( + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) + && + (_tmp_92_var = _tmp_92_rule(p)) + ) + { + res = _tmp_92_var; + goto done; + } + p->mark = mark; + } { // '...' void *literal; if ( @@ -7336,7 +7338,7 @@ strings_rule(Parser *p) { // STRING+ asdl_seq * a; if ( - (a = _loop1_92_rule(p)) + (a = _loop1_93_rule(p)) ) { res = _PyPegen_concatenate_strings ( p , a ); @@ -7494,7 +7496,7 @@ tuple_rule(Parser *p) if ( (literal = _PyPegen_expect_token(p, 7)) && - (a = _tmp_93_rule(p), 1) + (a = _tmp_94_rule(p), 1) && (literal_1 = _PyPegen_expect_token(p, 8)) ) @@ -7537,7 +7539,7 @@ group_rule(Parser *p) if ( (literal = _PyPegen_expect_token(p, 7)) && - (a = _tmp_94_rule(p)) + (a = _tmp_95_rule(p)) && (literal_1 = _PyPegen_expect_token(p, 8)) ) @@ -7856,7 +7858,7 @@ kvpairs_rule(Parser *p) void *opt_var; UNUSED(opt_var); // Silence compiler warnings if ( - (a = _gather_95_rule(p)) + (a = _gather_96_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -7940,7 +7942,7 @@ for_if_clauses_rule(Parser *p) { // ((ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*))+ asdl_seq * a; if ( - (a = _loop1_97_rule(p)) + (a = _loop1_98_rule(p)) ) { res = a; @@ -8106,7 +8108,7 @@ args_rule(Parser *p) if ( (a = starred_expression_rule(p)) && - (b = _tmp_98_rule(p), 1) + (b = _tmp_99_rule(p), 1) ) { Token *token = _PyPegen_get_last_nonnwhitespace_token(p); @@ -8155,7 +8157,7 @@ args_rule(Parser *p) if ( (a = named_expression_rule(p)) && - (b = _tmp_99_rule(p), 1) + (b = _tmp_100_rule(p), 1) ) { Token *token = _PyPegen_get_last_nonnwhitespace_token(p); @@ -8197,11 +8199,11 @@ kwargs_rule(Parser *p) asdl_seq * b; void *literal; if ( - (a = _gather_100_rule(p)) + (a = _gather_101_rule(p)) && (literal = _PyPegen_expect_token(p, 12)) && - (b = _gather_102_rule(p)) + (b = _gather_103_rule(p)) ) { res = _PyPegen_join_sequences ( p , a , b ); @@ -8214,23 +8216,23 @@ kwargs_rule(Parser *p) p->mark = mark; } { // ','.kwarg_or_starred+ - asdl_seq * _gather_104_var; + asdl_seq * _gather_105_var; if ( - (_gather_104_var = _gather_104_rule(p)) + (_gather_105_var = _gather_105_rule(p)) ) { - res = _gather_104_var; + res = _gather_105_var; goto done; } p->mark = mark; } { // ','.kwarg_or_double_starred+ - asdl_seq * _gather_106_var; + asdl_seq * _gather_107_var; if ( - (_gather_106_var = _gather_106_rule(p)) + (_gather_107_var = _gather_107_rule(p)) ) { - res = _gather_106_var; + res = _gather_107_var; goto done; } p->mark = mark; @@ -8473,7 +8475,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) && - (b = _loop0_108_rule(p)) + (b = _loop0_109_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -8514,7 +8516,7 @@ star_targets_seq_rule(Parser *p) void *opt_var; UNUSED(opt_var); // Silence compiler warnings if ( - (a = _gather_109_rule(p)) + (a = _gather_110_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -8562,7 +8564,7 @@ star_target_rule(Parser *p) if ( (literal = _PyPegen_expect_token(p, 16)) && - (a = _tmp_111_rule(p)) + (a = _tmp_112_rule(p)) ) { Token *token = _PyPegen_get_last_nonnwhitespace_token(p); @@ -8951,7 +8953,7 @@ del_targets_rule(Parser *p) void *opt_var; UNUSED(opt_var); // Silence compiler warnings if ( - (a = _gather_112_rule(p)) + (a = _gather_113_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -9204,7 +9206,7 @@ targets_rule(Parser *p) void *opt_var; UNUSED(opt_var); // Silence compiler warnings if ( - (a = _gather_114_rule(p)) + (a = _gather_115_rule(p)) && (opt_var = _PyPegen_expect_token(p, 12), 1) ) @@ -9732,7 +9734,7 @@ incorrect_arguments_rule(Parser *p) && (literal = _PyPegen_expect_token(p, 12)) && - (opt_var = _tmp_116_rule(p), 1) + (opt_var = _tmp_117_rule(p), 1) ) { res = RAISE_SYNTAX_ERROR ( "Generator expression must be parenthesized" ); @@ -9867,7 +9869,7 @@ invalid_assignment_rule(Parser *p) && (expression_var_1 = expression_rule(p)) && - (opt_var = _tmp_117_rule(p), 1) + (opt_var = _tmp_118_rule(p), 1) ) { res = RAISE_SYNTAX_ERROR ( "illegal target for annotation" ); @@ -9880,15 +9882,15 @@ invalid_assignment_rule(Parser *p) p->mark = mark; } { // expression ('=' | augassign) (yield_expr | star_expressions) - void *_tmp_118_var; void *_tmp_119_var; + void *_tmp_120_var; expr_ty a; if ( (a = expression_rule(p)) && - (_tmp_118_var = _tmp_118_rule(p)) - && (_tmp_119_var = _tmp_119_rule(p)) + && + (_tmp_120_var = _tmp_120_rule(p)) ) { res = RAISE_SYNTAX_ERROR ( "cannot assign to %s" , _PyPegen_get_expr_name ( a ) ); @@ -9946,12 +9948,12 @@ invalid_comprehension_rule(Parser *p) void * res = NULL; int mark = p->mark; { // ('[' | '(' | '{') '*' expression for_if_clauses - void *_tmp_120_var; + void *_tmp_121_var; expr_ty expression_var; asdl_seq* for_if_clauses_var; void *literal; if ( - (_tmp_120_var = _tmp_120_rule(p)) + (_tmp_121_var = _tmp_121_rule(p)) && (literal = _PyPegen_expect_token(p, 16)) && @@ -9985,15 +9987,15 @@ invalid_parameters_rule(Parser *p) void * res = NULL; int mark = p->mark; { // [plain_names ','] (slash_with_default | names_with_default) ',' plain_names - void *_tmp_122_var; + void *_tmp_123_var; void *literal; void *opt_var; UNUSED(opt_var); // Silence compiler warnings asdl_seq* plain_names_var; if ( - (opt_var = _tmp_121_rule(p), 1) + (opt_var = _tmp_122_rule(p), 1) && - (_tmp_122_var = _tmp_122_rule(p)) + (_tmp_123_var = _tmp_123_rule(p)) && (literal = _PyPegen_expect_token(p, 12)) && @@ -10520,12 +10522,12 @@ _loop1_13_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // (star_targets '=') - void *_tmp_123_var; + void *_tmp_124_var; while ( - (_tmp_123_var = _tmp_123_rule(p)) + (_tmp_124_var = _tmp_124_rule(p)) ) { - res = _tmp_123_var; + res = _tmp_124_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -10847,12 +10849,12 @@ _loop0_21_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('.' | '...') - void *_tmp_124_var; + void *_tmp_125_var; while ( - (_tmp_124_var = _tmp_124_rule(p)) + (_tmp_125_var = _tmp_125_rule(p)) ) { - res = _tmp_124_var; + res = _tmp_125_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -10896,12 +10898,12 @@ _loop1_22_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('.' | '...') - void *_tmp_125_var; + void *_tmp_126_var; while ( - (_tmp_125_var = _tmp_125_rule(p)) + (_tmp_126_var = _tmp_126_rule(p)) ) { - res = _tmp_125_var; + res = _tmp_126_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -12110,7 +12112,7 @@ _loop0_55_rule(Parser *p) while ( (literal = _PyPegen_expect_token(p, 12)) && - (elem = _tmp_126_rule(p)) + (elem = _tmp_127_rule(p)) ) { res = elem; @@ -12157,7 +12159,7 @@ _gather_54_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_126_rule(p)) + (elem = _tmp_127_rule(p)) && (seq = _loop0_55_rule(p)) ) @@ -12222,12 +12224,12 @@ _loop1_57_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('@' named_expression NEWLINE) - void *_tmp_127_var; + void *_tmp_128_var; while ( - (_tmp_127_var = _tmp_127_rule(p)) + (_tmp_128_var = _tmp_128_rule(p)) ) { - res = _tmp_127_var; + res = _tmp_128_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -12395,12 +12397,12 @@ _loop1_61_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // (',' star_expression) - void *_tmp_128_var; + void *_tmp_129_var; while ( - (_tmp_128_var = _tmp_128_rule(p)) + (_tmp_129_var = _tmp_129_rule(p)) ) { - res = _tmp_128_var; + res = _tmp_129_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -12533,12 +12535,12 @@ _loop1_64_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // (',' expression) - void *_tmp_129_var; + void *_tmp_130_var; while ( - (_tmp_129_var = _tmp_129_rule(p)) + (_tmp_130_var = _tmp_130_rule(p)) ) { - res = _tmp_129_var; + res = _tmp_130_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -13162,7 +13164,7 @@ _loop0_82_rule(Parser *p) while ( (literal = _PyPegen_expect_token(p, 12)) && - (elem = _tmp_130_rule(p)) + (elem = _tmp_131_rule(p)) ) { res = elem; @@ -13209,7 +13211,7 @@ _gather_81_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_130_rule(p)) + (elem = _tmp_131_rule(p)) && (seq = _loop0_82_rule(p)) ) @@ -13242,12 +13244,12 @@ _loop1_83_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('or' conjunction) - void *_tmp_131_var; + void *_tmp_132_var; while ( - (_tmp_131_var = _tmp_131_rule(p)) + (_tmp_132_var = _tmp_132_rule(p)) ) { - res = _tmp_131_var; + res = _tmp_132_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -13295,12 +13297,12 @@ _loop1_84_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('and' inversion) - void *_tmp_132_var; + void *_tmp_133_var; while ( - (_tmp_132_var = _tmp_132_rule(p)) + (_tmp_133_var = _tmp_133_rule(p)) ) { - res = _tmp_132_var; + res = _tmp_133_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -13383,9 +13385,38 @@ _loop1_85_rule(Parser *p) return seq; } -// _loop0_87: ',' slice +// _tmp_86: '!=' +static void * +_tmp_86_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '!=' + void *tok; + if ( + (tok = _PyPegen_expect_token(p, 28)) + ) + { + res = _PyPegen_check_barry_as_flufl ( p ) ? NULL : tok; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_88: ',' slice static asdl_seq * -_loop0_87_rule(Parser *p) +_loop0_88_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13430,32 +13461,32 @@ _loop0_87_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_87"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_88"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_87_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_88_type, seq); return seq; } -// _gather_86: slice _loop0_87 +// _gather_87: slice _loop0_88 static asdl_seq * -_gather_86_rule(Parser *p) +_gather_87_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // slice _loop0_87 + { // slice _loop0_88 expr_ty elem; asdl_seq * seq; if ( (elem = slice_rule(p)) && - (seq = _loop0_87_rule(p)) + (seq = _loop0_88_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -13468,9 +13499,9 @@ _gather_86_rule(Parser *p) return res; } -// _tmp_88: ':' expression? +// _tmp_89: ':' expression? static void * -_tmp_88_rule(Parser *p) +_tmp_89_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13500,9 +13531,9 @@ _tmp_88_rule(Parser *p) return res; } -// _tmp_89: tuple | group | genexp +// _tmp_90: tuple | group | genexp static void * -_tmp_89_rule(Parser *p) +_tmp_90_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13547,9 +13578,9 @@ _tmp_89_rule(Parser *p) return res; } -// _tmp_90: list | listcomp +// _tmp_91: list | listcomp static void * -_tmp_90_rule(Parser *p) +_tmp_91_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13583,9 +13614,9 @@ _tmp_90_rule(Parser *p) return res; } -// _tmp_91: dict | set | dictcomp | setcomp +// _tmp_92: dict | set | dictcomp | setcomp static void * -_tmp_91_rule(Parser *p) +_tmp_92_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13641,9 +13672,9 @@ _tmp_91_rule(Parser *p) return res; } -// _loop1_92: STRING +// _loop1_93: STRING static asdl_seq * -_loop1_92_rule(Parser *p) +_loop1_93_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13684,19 +13715,19 @@ _loop1_92_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_92"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_93"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop1_92_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop1_93_type, seq); return seq; } -// _tmp_93: star_named_expression ',' star_named_expressions? +// _tmp_94: star_named_expression ',' star_named_expressions? static void * -_tmp_93_rule(Parser *p) +_tmp_94_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13729,9 +13760,9 @@ _tmp_93_rule(Parser *p) return res; } -// _tmp_94: yield_expr | named_expression +// _tmp_95: yield_expr | named_expression static void * -_tmp_94_rule(Parser *p) +_tmp_95_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13765,9 +13796,9 @@ _tmp_94_rule(Parser *p) return res; } -// _loop0_96: ',' kvpair +// _loop0_97: ',' kvpair static asdl_seq * -_loop0_96_rule(Parser *p) +_loop0_97_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13812,32 +13843,32 @@ _loop0_96_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_96"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_97"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_96_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_97_type, seq); return seq; } -// _gather_95: kvpair _loop0_96 +// _gather_96: kvpair _loop0_97 static asdl_seq * -_gather_95_rule(Parser *p) +_gather_96_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // kvpair _loop0_96 + { // kvpair _loop0_97 KeyValuePair* elem; asdl_seq * seq; if ( (elem = kvpair_rule(p)) && - (seq = _loop0_96_rule(p)) + (seq = _loop0_97_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -13850,9 +13881,9 @@ _gather_95_rule(Parser *p) return res; } -// _loop1_97: (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*) +// _loop1_98: (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*) static asdl_seq * -_loop1_97_rule(Parser *p) +_loop1_98_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -13868,12 +13899,12 @@ _loop1_97_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*) - void *_tmp_133_var; + void *_tmp_134_var; while ( - (_tmp_133_var = _tmp_133_rule(p)) + (_tmp_134_var = _tmp_134_rule(p)) ) { - res = _tmp_133_var; + res = _tmp_134_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -13893,48 +13924,16 @@ _loop1_97_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_97"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_98"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop1_97_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop1_98_type, seq); return seq; } -// _tmp_98: ',' args -static void * -_tmp_98_rule(Parser *p) -{ - if (p->error_indicator) { - return NULL; - } - void * res = NULL; - int mark = p->mark; - { // ',' args - expr_ty c; - void *literal; - if ( - (literal = _PyPegen_expect_token(p, 12)) - && - (c = args_rule(p)) - ) - { - res = c; - if (res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - return NULL; - } - goto done; - } - p->mark = mark; - } - res = NULL; - done: - return res; -} - // _tmp_99: ',' args static void * _tmp_99_rule(Parser *p) @@ -13967,9 +13966,41 @@ _tmp_99_rule(Parser *p) return res; } -// _loop0_101: ',' kwarg_or_starred +// _tmp_100: ',' args +static void * +_tmp_100_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // ',' args + expr_ty c; + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 12)) + && + (c = args_rule(p)) + ) + { + res = c; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_102: ',' kwarg_or_starred static asdl_seq * -_loop0_101_rule(Parser *p) +_loop0_102_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14014,32 +14045,32 @@ _loop0_101_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_101"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_102"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_101_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_102_type, seq); return seq; } -// _gather_100: kwarg_or_starred _loop0_101 +// _gather_101: kwarg_or_starred _loop0_102 static asdl_seq * -_gather_100_rule(Parser *p) +_gather_101_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // kwarg_or_starred _loop0_101 + { // kwarg_or_starred _loop0_102 KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) && - (seq = _loop0_101_rule(p)) + (seq = _loop0_102_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14052,9 +14083,9 @@ _gather_100_rule(Parser *p) return res; } -// _loop0_103: ',' kwarg_or_double_starred +// _loop0_104: ',' kwarg_or_double_starred static asdl_seq * -_loop0_103_rule(Parser *p) +_loop0_104_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14099,32 +14130,32 @@ _loop0_103_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_103"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_104"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_103_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_104_type, seq); return seq; } -// _gather_102: kwarg_or_double_starred _loop0_103 +// _gather_103: kwarg_or_double_starred _loop0_104 static asdl_seq * -_gather_102_rule(Parser *p) +_gather_103_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // kwarg_or_double_starred _loop0_103 + { // kwarg_or_double_starred _loop0_104 KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) && - (seq = _loop0_103_rule(p)) + (seq = _loop0_104_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14137,9 +14168,9 @@ _gather_102_rule(Parser *p) return res; } -// _loop0_105: ',' kwarg_or_starred +// _loop0_106: ',' kwarg_or_starred static asdl_seq * -_loop0_105_rule(Parser *p) +_loop0_106_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14184,32 +14215,32 @@ _loop0_105_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_105"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_106"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_105_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_106_type, seq); return seq; } -// _gather_104: kwarg_or_starred _loop0_105 +// _gather_105: kwarg_or_starred _loop0_106 static asdl_seq * -_gather_104_rule(Parser *p) +_gather_105_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // kwarg_or_starred _loop0_105 + { // kwarg_or_starred _loop0_106 KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) && - (seq = _loop0_105_rule(p)) + (seq = _loop0_106_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14222,92 +14253,7 @@ _gather_104_rule(Parser *p) return res; } -// _loop0_107: ',' kwarg_or_double_starred -static asdl_seq * -_loop0_107_rule(Parser *p) -{ - if (p->error_indicator) { - return NULL; - } - void *res = NULL; - int mark = p->mark; - int start_mark = p->mark; - void **children = PyMem_Malloc(sizeof(void *)); - if (!children) { - PyErr_Format(PyExc_MemoryError, "Parser out of memory"); - return NULL; - } - ssize_t children_capacity = 1; - ssize_t n = 0; - { // ',' kwarg_or_double_starred - KeywordOrStarred* elem; - void *literal; - while ( - (literal = _PyPegen_expect_token(p, 12)) - && - (elem = kwarg_or_double_starred_rule(p)) - ) - { - res = elem; - if (res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - PyMem_Free(children); - return NULL; - } - if (n == children_capacity) { - children_capacity *= 2; - children = PyMem_Realloc(children, children_capacity*sizeof(void *)); - if (!children) { - PyErr_Format(PyExc_MemoryError, "realloc None"); - return NULL; - } - } - children[n++] = res; - mark = p->mark; - } - p->mark = mark; - } - asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); - if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_107"); - PyMem_Free(children); - return NULL; - } - for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); - PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_107_type, seq); - return seq; -} - -// _gather_106: kwarg_or_double_starred _loop0_107 -static asdl_seq * -_gather_106_rule(Parser *p) -{ - if (p->error_indicator) { - return NULL; - } - asdl_seq * res = NULL; - int mark = p->mark; - { // kwarg_or_double_starred _loop0_107 - KeywordOrStarred* elem; - asdl_seq * seq; - if ( - (elem = kwarg_or_double_starred_rule(p)) - && - (seq = _loop0_107_rule(p)) - ) - { - res = _PyPegen_seq_insert_in_front(p, elem, seq); - goto done; - } - p->mark = mark; - } - res = NULL; - done: - return res; -} - -// _loop0_108: (',' star_target) +// _loop0_108: ',' kwarg_or_double_starred static asdl_seq * _loop0_108_rule(Parser *p) { @@ -14324,13 +14270,21 @@ _loop0_108_rule(Parser *p) } ssize_t children_capacity = 1; ssize_t n = 0; - { // (',' star_target) - void *_tmp_134_var; + { // ',' kwarg_or_double_starred + KeywordOrStarred* elem; + void *literal; while ( - (_tmp_134_var = _tmp_134_rule(p)) + (literal = _PyPegen_expect_token(p, 12)) + && + (elem = kwarg_or_double_starred_rule(p)) ) { - res = _tmp_134_var; + res = elem; + if (res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + PyMem_Free(children); + return NULL; + } if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -14356,9 +14310,86 @@ _loop0_108_rule(Parser *p) return seq; } -// _loop0_110: ',' star_target +// _gather_107: kwarg_or_double_starred _loop0_108 static asdl_seq * -_loop0_110_rule(Parser *p) +_gather_107_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + asdl_seq * res = NULL; + int mark = p->mark; + { // kwarg_or_double_starred _loop0_108 + KeywordOrStarred* elem; + asdl_seq * seq; + if ( + (elem = kwarg_or_double_starred_rule(p)) + && + (seq = _loop0_108_rule(p)) + ) + { + res = _PyPegen_seq_insert_in_front(p, elem, seq); + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _loop0_109: (',' star_target) +static asdl_seq * +_loop0_109_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void *res = NULL; + int mark = p->mark; + int start_mark = p->mark; + void **children = PyMem_Malloc(sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "Parser out of memory"); + return NULL; + } + ssize_t children_capacity = 1; + ssize_t n = 0; + { // (',' star_target) + void *_tmp_135_var; + while ( + (_tmp_135_var = _tmp_135_rule(p)) + ) + { + res = _tmp_135_var; + if (n == children_capacity) { + children_capacity *= 2; + children = PyMem_Realloc(children, children_capacity*sizeof(void *)); + if (!children) { + PyErr_Format(PyExc_MemoryError, "realloc None"); + return NULL; + } + } + children[n++] = res; + mark = p->mark; + } + p->mark = mark; + } + asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); + if (!seq) { + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_109"); + PyMem_Free(children); + return NULL; + } + for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); + PyMem_Free(children); + _PyPegen_insert_memo(p, start_mark, _loop0_109_type, seq); + return seq; +} + +// _loop0_111: ',' star_target +static asdl_seq * +_loop0_111_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14403,32 +14434,32 @@ _loop0_110_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_110"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_111"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_110_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_111_type, seq); return seq; } -// _gather_109: star_target _loop0_110 +// _gather_110: star_target _loop0_111 static asdl_seq * -_gather_109_rule(Parser *p) +_gather_110_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // star_target _loop0_110 + { // star_target _loop0_111 expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) && - (seq = _loop0_110_rule(p)) + (seq = _loop0_111_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14441,9 +14472,9 @@ _gather_109_rule(Parser *p) return res; } -// _tmp_111: !'*' star_target +// _tmp_112: !'*' star_target static void * -_tmp_111_rule(Parser *p) +_tmp_112_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14468,9 +14499,9 @@ _tmp_111_rule(Parser *p) return res; } -// _loop0_113: ',' del_target +// _loop0_114: ',' del_target static asdl_seq * -_loop0_113_rule(Parser *p) +_loop0_114_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14515,32 +14546,32 @@ _loop0_113_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_113"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_114"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_113_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_114_type, seq); return seq; } -// _gather_112: del_target _loop0_113 +// _gather_113: del_target _loop0_114 static asdl_seq * -_gather_112_rule(Parser *p) +_gather_113_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // del_target _loop0_113 + { // del_target _loop0_114 expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) && - (seq = _loop0_113_rule(p)) + (seq = _loop0_114_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14553,9 +14584,9 @@ _gather_112_rule(Parser *p) return res; } -// _loop0_115: ',' target +// _loop0_116: ',' target static asdl_seq * -_loop0_115_rule(Parser *p) +_loop0_116_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14600,32 +14631,32 @@ _loop0_115_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_115"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_116"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_115_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_116_type, seq); return seq; } -// _gather_114: target _loop0_115 +// _gather_115: target _loop0_116 static asdl_seq * -_gather_114_rule(Parser *p) +_gather_115_rule(Parser *p) { if (p->error_indicator) { return NULL; } asdl_seq * res = NULL; int mark = p->mark; - { // target _loop0_115 + { // target _loop0_116 expr_ty elem; asdl_seq * seq; if ( (elem = target_rule(p)) && - (seq = _loop0_115_rule(p)) + (seq = _loop0_116_rule(p)) ) { res = _PyPegen_seq_insert_in_front(p, elem, seq); @@ -14638,9 +14669,9 @@ _gather_114_rule(Parser *p) return res; } -// _tmp_116: args | expression for_if_clauses +// _tmp_117: args | expression for_if_clauses static void * -_tmp_116_rule(Parser *p) +_tmp_117_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14677,9 +14708,9 @@ _tmp_116_rule(Parser *p) return res; } -// _tmp_117: '=' annotated_rhs +// _tmp_118: '=' annotated_rhs static void * -_tmp_117_rule(Parser *p) +_tmp_118_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14705,9 +14736,9 @@ _tmp_117_rule(Parser *p) return res; } -// _tmp_118: '=' | augassign +// _tmp_119: '=' | augassign static void * -_tmp_118_rule(Parser *p) +_tmp_119_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14741,9 +14772,9 @@ _tmp_118_rule(Parser *p) return res; } -// _tmp_119: yield_expr | star_expressions +// _tmp_120: yield_expr | star_expressions static void * -_tmp_119_rule(Parser *p) +_tmp_120_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14777,9 +14808,9 @@ _tmp_119_rule(Parser *p) return res; } -// _tmp_120: '[' | '(' | '{' +// _tmp_121: '[' | '(' | '{' static void * -_tmp_120_rule(Parser *p) +_tmp_121_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14824,9 +14855,9 @@ _tmp_120_rule(Parser *p) return res; } -// _tmp_121: plain_names ',' +// _tmp_122: plain_names ',' static void * -_tmp_121_rule(Parser *p) +_tmp_122_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14852,9 +14883,9 @@ _tmp_121_rule(Parser *p) return res; } -// _tmp_122: slash_with_default | names_with_default +// _tmp_123: slash_with_default | names_with_default static void * -_tmp_122_rule(Parser *p) +_tmp_123_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14888,9 +14919,9 @@ _tmp_122_rule(Parser *p) return res; } -// _tmp_123: star_targets '=' +// _tmp_124: star_targets '=' static void * -_tmp_123_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -14920,42 +14951,6 @@ _tmp_123_rule(Parser *p) return res; } -// _tmp_124: '.' | '...' -static void * -_tmp_124_rule(Parser *p) -{ - if (p->error_indicator) { - return NULL; - } - void * res = NULL; - int mark = p->mark; - { // '.' - void *literal; - if ( - (literal = _PyPegen_expect_token(p, 23)) - ) - { - res = literal; - goto done; - } - p->mark = mark; - } - { // '...' - void *literal; - if ( - (literal = _PyPegen_expect_token(p, 52)) - ) - { - res = literal; - goto done; - } - p->mark = mark; - } - res = NULL; - done: - return res; -} - // _tmp_125: '.' | '...' static void * _tmp_125_rule(Parser *p) @@ -14992,9 +14987,45 @@ _tmp_125_rule(Parser *p) return res; } -// _tmp_126: plain_name !'=' +// _tmp_126: '.' | '...' static void * _tmp_126_rule(Parser *p) +{ + if (p->error_indicator) { + return NULL; + } + void * res = NULL; + int mark = p->mark; + { // '.' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 23)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + { // '...' + void *literal; + if ( + (literal = _PyPegen_expect_token(p, 52)) + ) + { + res = literal; + goto done; + } + p->mark = mark; + } + res = NULL; + done: + return res; +} + +// _tmp_127: plain_name !'=' +static void * +_tmp_127_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15019,9 +15050,9 @@ _tmp_126_rule(Parser *p) return res; } -// _tmp_127: '@' named_expression NEWLINE +// _tmp_128: '@' named_expression NEWLINE static void * -_tmp_127_rule(Parser *p) +_tmp_128_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15054,9 +15085,9 @@ _tmp_127_rule(Parser *p) return res; } -// _tmp_128: ',' star_expression +// _tmp_129: ',' star_expression static void * -_tmp_128_rule(Parser *p) +_tmp_129_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15086,9 +15117,9 @@ _tmp_128_rule(Parser *p) return res; } -// _tmp_129: ',' expression +// _tmp_130: ',' expression static void * -_tmp_129_rule(Parser *p) +_tmp_130_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15118,9 +15149,9 @@ _tmp_129_rule(Parser *p) return res; } -// _tmp_130: lambda_plain_name !'=' +// _tmp_131: lambda_plain_name !'=' static void * -_tmp_130_rule(Parser *p) +_tmp_131_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15145,9 +15176,9 @@ _tmp_130_rule(Parser *p) return res; } -// _tmp_131: 'or' conjunction +// _tmp_132: 'or' conjunction static void * -_tmp_131_rule(Parser *p) +_tmp_132_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15177,9 +15208,9 @@ _tmp_131_rule(Parser *p) return res; } -// _tmp_132: 'and' inversion +// _tmp_133: 'and' inversion static void * -_tmp_132_rule(Parser *p) +_tmp_133_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15209,9 +15240,9 @@ _tmp_132_rule(Parser *p) return res; } -// _tmp_133: ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))* +// _tmp_134: ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))* static void * -_tmp_133_rule(Parser *p) +_tmp_134_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15236,7 +15267,7 @@ _tmp_133_rule(Parser *p) && (b = disjunction_rule(p)) && - (c = _loop0_135_rule(p)) + (c = _loop0_136_rule(p)) ) { res = _Py_comprehension ( a , b , c , y != NULL , p -> arena ); @@ -15253,9 +15284,9 @@ _tmp_133_rule(Parser *p) return res; } -// _tmp_134: ',' star_target +// _tmp_135: ',' star_target static void * -_tmp_134_rule(Parser *p) +_tmp_135_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15285,9 +15316,9 @@ _tmp_134_rule(Parser *p) return res; } -// _loop0_135: ('if' disjunction) +// _loop0_136: ('if' disjunction) static asdl_seq * -_loop0_135_rule(Parser *p) +_loop0_136_rule(Parser *p) { if (p->error_indicator) { return NULL; @@ -15303,12 +15334,12 @@ _loop0_135_rule(Parser *p) ssize_t children_capacity = 1; ssize_t n = 0; { // ('if' disjunction) - void *_tmp_136_var; + void *_tmp_137_var; while ( - (_tmp_136_var = _tmp_136_rule(p)) + (_tmp_137_var = _tmp_137_rule(p)) ) { - res = _tmp_136_var; + res = _tmp_137_var; if (n == children_capacity) { children_capacity *= 2; children = PyMem_Realloc(children, children_capacity*sizeof(void *)); @@ -15324,19 +15355,19 @@ _loop0_135_rule(Parser *p) } asdl_seq *seq = _Py_asdl_seq_new(n, p->arena); if (!seq) { - PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_135"); + PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_136"); PyMem_Free(children); return NULL; } for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]); PyMem_Free(children); - _PyPegen_insert_memo(p, start_mark, _loop0_135_type, seq); + _PyPegen_insert_memo(p, start_mark, _loop0_136_type, seq); return seq; } -// _tmp_136: 'if' disjunction +// _tmp_137: 'if' disjunction static void * -_tmp_136_rule(Parser *p) +_tmp_137_rule(Parser *p) { if (p->error_indicator) { return NULL; diff --git a/Parser/pegen/parse_string.c b/Parser/pegen/parse_string.c index 41485a9669d..9a78a28d241 100644 --- a/Parser/pegen/parse_string.c +++ b/Parser/pegen/parse_string.c @@ -586,7 +586,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, return NULL; } - Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, NULL, p->arena); + Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, p->flags, NULL, p->arena); p2->starting_lineno = p->starting_lineno + p->tok->first_lineno - 1; p2->starting_col_offset = p->tok->first_lineno == p->tok->lineno ? p->starting_col_offset + t->col_offset : 0; diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c index c42aa680c86..31ac2e13992 100644 --- a/Parser/pegen/peg_api.c +++ b/Parser/pegen/peg_api.c @@ -22,20 +22,19 @@ PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, PyCom return NULL; } - int iflags = flags != NULL ? flags->cf_flags : PyCF_IGNORE_COOKIE; - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, iflags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); return result; } mod_ty -PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena) +PyPegen_ASTFromFile(const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena) { PyObject *filename_ob = PyUnicode_FromString(filename); if (filename_ob == NULL) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_file(filename, mode, filename_ob, arena); + mod_ty result = _PyPegen_run_parser_from_file(filename, mode, filename_ob, flags, arena); Py_XDECREF(filename_ob); return result; } @@ -43,13 +42,13 @@ PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena) mod_ty PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, int mode, const char *enc, const char *ps1, const char* ps2, - int *errcode, PyArena *arena) + PyCompilerFlags *flags, int *errcode, PyArena *arena) { if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) { return NULL; } return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2, - errcode, arena); + flags, errcode, arena); } PyCodeObject * @@ -81,7 +80,7 @@ error: } PyCodeObject * -PyPegen_CodeObjectFromFile(const char *filename, int mode) +PyPegen_CodeObjectFromFile(const char *filename, int mode, PyCompilerFlags* flags) { PyArena *arena = PyArena_New(); if (arena == NULL) { @@ -95,7 +94,7 @@ PyPegen_CodeObjectFromFile(const char *filename, int mode) goto error; } - mod_ty res = PyPegen_ASTFromFile(filename, mode, arena); + mod_ty res = PyPegen_ASTFromFile(filename, mode, flags, arena); if (res == NULL) { goto error; } @@ -110,8 +109,8 @@ error: PyCodeObject * PyPegen_CodeObjectFromFileObject(FILE *fp, PyObject *filename_ob, int mode, - const char *ps1, const char *ps2, const char *enc, - int *errcode) + const char *ps1, const char *ps2, + PyCompilerFlags *flags, const char *enc, int *errcode) { PyArena *arena = PyArena_New(); if (arena == NULL) { @@ -121,7 +120,7 @@ PyPegen_CodeObjectFromFileObject(FILE *fp, PyObject *filename_ob, int mode, PyCodeObject *result = NULL; mod_ty res = PyPegen_ASTFromFileObject(fp, filename_ob, mode, enc, ps1, ps2, - errcode, arena); + flags, errcode, arena); if (res == NULL) { goto error; } diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index c8f5c95b473..081da8238c3 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -25,6 +25,24 @@ init_normalization(Parser *p) return 1; } +/* Checks if the NOTEQUAL token is valid given the current parser flags +0 indicates success and nonzero indicates failure (an exception may be set) */ +int +_PyPegen_check_barry_as_flufl(Parser *p) { + Token *t = p->tokens[p->fill - 1]; + assert(t->bytes != NULL); + assert(t->type == NOTEQUAL); + + char* tok_str = PyBytes_AS_STRING(t->bytes); + if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>")){ + RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='"); + return -1; + } else if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) { + return strcmp(tok_str, "!="); + } + return 0; +} + PyObject * _PyPegen_new_identifier(Parser *p, char *n) { @@ -401,7 +419,6 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...) loc = Py_None; } - tmp = Py_BuildValue("(OiiN)", p->tok->filename, t->lineno, col_number, loc); if (!tmp) { goto error; @@ -902,8 +919,31 @@ _PyPegen_Parser_Free(Parser *p) PyMem_Free(p); } +static int +compute_parser_flags(PyCompilerFlags *flags) +{ + int parser_flags = 0; + if (!flags) { + return 0; + } + if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) { + parser_flags |= PyPARSE_DONT_IMPLY_DEDENT; + } + if (flags->cf_flags & PyCF_IGNORE_COOKIE) { + parser_flags |= PyPARSE_IGNORE_COOKIE; + } + if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) { + parser_flags |= PyPARSE_BARRY_AS_BDFL; + } + if (flags->cf_flags & PyCF_TYPE_COMMENTS) { + parser_flags |= PyPARSE_TYPE_COMMENTS; + } + return parser_flags; +} + Parser * -_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena *arena) +_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, + int *errcode, PyArena *arena) { Parser *p = PyMem_Malloc(sizeof(Parser)); if (p == NULL) { @@ -938,6 +978,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena p->starting_lineno = 0; p->starting_col_offset = 0; + p->flags = flags; return p; } @@ -976,7 +1017,7 @@ _PyPegen_run_parser(Parser *p) mod_ty _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob, const char *enc, const char *ps1, const char *ps2, - int *errcode, PyArena *arena) + PyCompilerFlags *flags, int *errcode, PyArena *arena) { struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2); if (tok == NULL) { @@ -993,7 +1034,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; - Parser *p = _PyPegen_Parser_New(tok, start_rule, errcode, arena); + int parser_flags = compute_parser_flags(flags); + Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, errcode, arena); if (p == NULL) { goto error; } @@ -1008,7 +1050,7 @@ error: mod_ty _PyPegen_run_parser_from_file(const char *filename, int start_rule, - PyObject *filename_ob, PyArena *arena) + PyObject *filename_ob, PyCompilerFlags *flags, PyArena *arena) { FILE *fp = fopen(filename, "rb"); if (fp == NULL) { @@ -1017,7 +1059,7 @@ _PyPegen_run_parser_from_file(const char *filename, int start_rule, } mod_ty result = _PyPegen_run_parser_from_file_pointer(fp, start_rule, filename_ob, - NULL, NULL, NULL, NULL, arena); + NULL, NULL, NULL, flags, NULL, arena); fclose(fp); return result; @@ -1025,12 +1067,12 @@ _PyPegen_run_parser_from_file(const char *filename, int start_rule, mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - int iflags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena) { int exec_input = start_rule == Py_file_input; struct tok_state *tok; - if (iflags & PyCF_IGNORE_COOKIE) { + if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) { tok = PyTokenizer_FromUTF8(str, exec_input); } else { tok = PyTokenizer_FromString(str, exec_input); @@ -1048,7 +1090,8 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen // We need to clear up from here on mod_ty result = NULL; - Parser *p = _PyPegen_Parser_New(tok, start_rule, NULL, arena); + int parser_flags = compute_parser_flags(flags); + Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, NULL, arena); if (p == NULL) { goto error; } diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h index a20ec4a0e42..0ac9b317efe 100644 --- a/Parser/pegen/pegen.h +++ b/Parser/pegen/pegen.h @@ -7,6 +7,23 @@ #include #include +#if 0 +#define PyPARSE_YIELD_IS_KEYWORD 0x0001 +#endif + +#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 + +#if 0 +#define PyPARSE_WITH_IS_KEYWORD 0x0003 +#define PyPARSE_PRINT_IS_FUNCTION 0x0004 +#define PyPARSE_UNICODE_LITERALS 0x0008 +#endif + +#define PyPARSE_IGNORE_COOKIE 0x0010 +#define PyPARSE_BARRY_AS_BDFL 0x0020 +#define PyPARSE_TYPE_COMMENTS 0x0040 +#define PyPARSE_ASYNC_HACKS 0x0080 + typedef struct _memo { int type; void *node; @@ -41,6 +58,7 @@ typedef struct { int starting_lineno; int starting_col_offset; int error_indicator; + int flags; } Parser; typedef struct { @@ -137,13 +155,13 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result) PyObject *_PyPegen_new_identifier(Parser *, char *); -Parser *_PyPegen_Parser_New(struct tok_state *, int, int *, PyArena *); +Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int *, PyArena *); void _PyPegen_Parser_Free(Parser *); mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, - const char *, const char *, int *, PyArena *); + const char *, const char *, PyCompilerFlags *, int *, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyArena *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, int, PyArena *); +mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); void *_PyPegen_interactive_exit(Parser *); asdl_seq *_PyPegen_singleton_seq(Parser *, void *); asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *); @@ -174,6 +192,7 @@ asdl_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *); asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); +int _PyPegen_check_barry_as_flufl(Parser *); void *_PyPegen_parse(Parser *); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 3a2fe966c08..79147e430a1 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -241,7 +241,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, if (use_peg) { mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input, - enc, ps1, ps2, &errcode, arena); + enc, ps1, ps2, flags, &errcode, arena); } else { mod = PyParser_ASTFromFileObject(fp, filename, enc, @@ -1073,7 +1073,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa if (use_peg) { mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL, - NULL, arena); + flags, NULL, arena); } else { mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0, diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index d8d36a0a1a5..fb552eed3ba 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -12,7 +12,6 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar } else { result = Py_None; Py_INCREF(result); - } return result; @@ -43,7 +42,8 @@ parse_file(PyObject *self, PyObject *args, PyObject *kwds) goto error; } - mod_ty res = _PyPegen_run_parser_from_file(filename, Py_file_input, filename_ob, arena); + PyCompilerFlags flags = _PyCompilerFlags_INIT; + mod_ty res = _PyPegen_run_parser_from_file(filename, Py_file_input, filename_ob, &flags, arena); if (res == NULL) { goto error; } @@ -81,8 +81,9 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) goto error; } + PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - PyCF_IGNORE_COOKIE, arena); + &flags, arena); if (res == NULL) { goto error; } diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 5b9d80453ca..6c4b8f1e7df 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -73,9 +73,17 @@ class CCallMakerVisitor(GrammarVisitor): return "literal", f"_PyPegen_expect_token(p, {type})" def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]: + def can_we_inline(node): + if len(node.alts) != 1 or len(node.alts[0].items) != 1: + return False + # If the alternative has an action we cannot inline + if getattr(node.alts[0], "action", None) is not None: + return False + return True + if node in self.cache: return self.cache[node] - if len(node.alts) == 1 and len(node.alts[0].items) == 1: + if can_we_inline(node): self.cache[node] = self.visit(node.alts[0].items[0]) else: name = self.gen.name_node(node)