bpo-40334: Rename PyConfig.use_peg to _use_peg_parser (GH-19670)

* Rename PyConfig.use_peg to _use_peg_parser
* Document PyConfig._use_peg_parser and mark it a deprecated
* Mark -X oldparser option and PYTHONOLDPARSER env var as deprecated
  in the documentation.
* Add use_old_parser() and skip_if_new_parser() to test.support
* Remove sys.flags.use_peg: use_old_parser() uses
  _testinternalcapi.get_configs() instead.
* Enhance test_embed tests
* subprocess._args_from_interpreter_flags() copies -X oldparser

Doc/c-api/init_config.rst

@@ -686,6 +686,16 @@ PyConfig
 
       :data:`sys._xoptions`.
 
+   .. c:member:: int _use_peg_parser
+
+      Enable PEG parser? Default: 1.
+
+      Set to 0 by :option:`-X oldparser <-X>` and :envvar:`PYTHONOLDPARSER`.
+
+      See also :pep:`617`.
+
+      .. deprecated-removed:: 3.9 3.10
+
 If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same
 way the regular Python parses command line arguments, and Python
 arguments are stripped from ``argv``: see :ref:`Command Line Arguments

Doc/using/cmdline.rst

@@ -427,7 +427,7 @@ Miscellaneous options
 
    * ``-X faulthandler`` to enable :mod:`faulthandler`;
    * ``-X oldparser``: enable the traditional LL(1) parser.  See also
-     :envvar:`PYTHONOLDPARSER`.
+     :envvar:`PYTHONOLDPARSER` and :pep:`617`.
    * ``-X showrefcount`` to output the total reference count and number of used
      memory blocks when the program finishes or after each statement in the
      interactive interpreter. This only works on debug builds.
@@ -480,6 +480,9 @@ Miscellaneous options
 
       The ``-X showalloccount`` option has been removed.
 
+   .. deprecated-removed:: 3.9 3.10
+      The ``-X oldparser`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -578,8 +581,11 @@ conflict.
 
 .. envvar:: PYTHONOLDPARSER
 
-   If this is set it is equivalent to specifying the :option:`-X`
+   If this is set to a non-empty string, enable the traditional LL(1) parser.
-   ``oldparser`` option.
+
+   See also the :option:`-X` ``oldparser`` option and :pep:`617`.
+
+   .. deprecated-removed:: 3.9 3.10
 
 
 .. envvar:: PYTHONINSPECT

Include/cpython/initconfig.h

@@ -149,7 +149,7 @@ typedef struct {
 
     /* Enable PEG parser?
        1 by default, set to 0 by -X oldparser and PYTHONOLDPARSER */
-    int use_peg;
+    int _use_peg_parser;
 
     /* Enable tracemalloc?
        Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */

Lib/subprocess.py

@@ -326,7 +326,7 @@ def _args_from_interpreter_flags():
     if dev_mode:
         args.extend(('-X', 'dev'))
     for opt in ('faulthandler', 'tracemalloc', 'importtime',
-                'showrefcount', 'utf8'):
+                'showrefcount', 'utf8', 'oldparser'):
         if opt in xoptions:
             value = xoptions[opt]
             if value is True:

Lib/test/support/__init__.py

@@ -3454,3 +3454,13 @@ def wait_process(pid, *, exitcode, timeout=None):
     # sanity check: it should not fail in practice
     if pid2 != pid:
         raise AssertionError(f"pid {pid2} != pid {pid}")
+
+
+def use_old_parser():
+    import _testinternalcapi
+    config = _testinternalcapi.get_configs()
+    return (config['config']['_use_peg_parser'] == 0)
+
+
+def skip_if_new_parser(msg):
+    return unittest.skipIf(not use_old_parser(), msg)

Lib/test/test_codeop.py

@@ -4,12 +4,12 @@
 """
 import sys
 import unittest
-from test.support import is_jython
+from test import support
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 import io
 
-if is_jython:
+if support.is_jython:
 
     def unify_callables(d):
         for n,v in d.items():
@@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase):
 
     def assertValid(self, str, symbol='single'):
         '''succeed iff str is a valid piece of code'''
-        if is_jython:
+        if support.is_jython:
             code = compile_command(str, "<input>", symbol)
             self.assertTrue(code)
             if symbol == "single":
@@ -60,7 +60,7 @@ class CodeopTests(unittest.TestCase):
         av = self.assertValid
 
         # special case
-        if not is_jython:
+        if not support.is_jython:
             self.assertEqual(compile_command(""),
                              compile("pass", "<input>", 'single',
                                      PyCF_DONT_IMPLY_DEDENT))
@@ -122,7 +122,7 @@ class CodeopTests(unittest.TestCase):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
+    @support.skip_if_new_parser("Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
     def test_incomplete(self):
         ai = self.assertIncomplete
 

Lib/test/test_compile.py

@@ -501,7 +501,7 @@ if 1:
         self.compile_single("if x:\n   f(x)\nelse:\n   g(x)")
         self.compile_single("class T:\n   pass")
 
-    @unittest.skipIf(sys.flags.use_peg, 'Pegen does not disallow multiline single stmts')
+    @support.skip_if_new_parser('Pegen does not disallow multiline single stmts')
     def test_bad_single_statement(self):
         self.assertInvalidSingle('1\n2')
         self.assertInvalidSingle('def f(): pass')

Lib/test/test_embed.py

@@ -347,7 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'isolated': 0,
         'use_environment': 1,
         'dev_mode': 0,
-        'use_peg': 1,
+        '_use_peg_parser': 1,
 
         'install_signal_handlers': 1,
         'use_hash_seed': 0,
@@ -729,7 +729,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'import_time': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
-            'use_peg': 0,
+            '_use_peg_parser': 0,
 
             'stdio_encoding': 'iso8859-1',
             'stdio_errors': 'replace',
@@ -792,6 +792,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_compat_env", config, preconfig,
                                api=API_COMPAT)
@@ -819,6 +820,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_python_env", config, preconfig,
                                api=API_PYTHON)

Lib/test/test_eof.py

@@ -26,7 +26,7 @@ class EOFTestCase(unittest.TestCase):
         else:
             raise support.TestFailed
 
-    @unittest.skipIf(sys.flags.use_peg, "TODO for PEG -- fails with new parser")
+    @support.skip_if_new_parser("TODO for PEG -- fails with new parser")
     def test_line_continuation_EOF(self):
         """A continuation at the end of input must be an error; bpo2180."""
         expect = 'unexpected EOF while parsing (<string>, line 1)'

Lib/test/test_exceptions.py

@@ -178,7 +178,7 @@ class ExceptionTests(unittest.TestCase):
         s = '''if True:\n        print()\n\texec "mixed tabs and spaces"'''
         ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError)
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen column offsets might be different")
+    @support.skip_if_new_parser("Pegen column offsets might be different")
     def testSyntaxErrorOffset(self):
         def check(src, lineno, offset, encoding='utf-8'):
             with self.assertRaises(SyntaxError) as cm:

Lib/test/test_flufl.py

@@ -1,9 +1,10 @@
 import __future__
 import unittest
 import sys
+from test import support
 
 
-@unittest.skipIf(sys.flags.use_peg, "Not supported by pegen yet")
+@support.skip_if_new_parser("Not supported by pegen yet")
 class FLUFLTests(unittest.TestCase):
 
     def test_barry_as_bdfl(self):

Lib/test/test_fstring.py

@@ -12,6 +12,7 @@ import types
 import decimal
 import sys
 import unittest
+from test import support
 
 a_global = 'global variable'
 
@@ -206,7 +207,7 @@ f'{a * f"-{x()}-"}'"""
         call = binop.right.values[1].value
         self.assertEqual(type(call), ast.Call)
         self.assertEqual(call.lineno, 3)
-        if not sys.flags.use_peg:
+        if support.use_old_parser():
             self.assertEqual(call.col_offset, 11)
 
     def test_ast_line_numbers_duplicate_expression(self):

Lib/test/test_parser.py

@@ -900,7 +900,7 @@ class ParserStackLimitTestCase(unittest.TestCase):
         st = parser.expr(e)
         st.compile()
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not trigger memory error with this many parenthesis")
+    @support.skip_if_new_parser("Pegen does not trigger memory error with this many parenthesis")
     def test_trigger_memory_error(self):
         e = self._nested_expression(100)
         rc, out, err = assert_python_failure('-Xoldparser', '-c', e)

Lib/test/test_peg_parser.py

@@ -6,6 +6,7 @@ import unittest
 from pathlib import PurePath
 from typing import Any, Union, Iterable, Tuple
 from textwrap import dedent
+from test import support
 
 
 TEST_CASES = [
@@ -720,7 +721,7 @@ class ASTGenerationTest(unittest.TestCase):
                 f"Actual error message does not match expexted for {source}"
             )
 
-    @unittest.skipIf(sys.flags.use_peg, "This tests nothing for now, since compile uses pegen as well")
+    @support.skip_if_new_parser("This tests nothing for now, since compile uses pegen as well")
     @unittest.expectedFailure
     def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:

Lib/test/test_positional_only_arg.py

@@ -5,7 +5,7 @@ import pickle
 import unittest
 import sys
 
-from test.support import check_syntax_error
+from test.support import check_syntax_error, use_old_parser
 
 
 def global_pos_only_f(a, b, /):
@@ -24,7 +24,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
             compile(codestr + "\n", "<test>", "single")
 
     def test_invalid_syntax_errors(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument")
@@ -47,7 +47,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
         check_syntax_error(self, "def f(a, *, c, /, d, e): pass")
 
     def test_invalid_syntax_errors_async(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
             check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument")
@@ -236,7 +236,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
         self.assertEqual(x(1, 2), 3)
 
     def test_invalid_syntax_lambda(self):
-        if not sys.flags.use_peg:
+        if use_old_parser():
             check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument")
             check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument")

Lib/test/test_string_literals.py

@@ -33,6 +33,7 @@ import shutil
 import tempfile
 import unittest
 import warnings
+from test.support import check_syntax_warning, use_old_parser
 
 
 TEMPLATE = r"""# coding: %s
@@ -63,8 +64,6 @@ def byte(i):
 
 class TestLiterals(unittest.TestCase):
 
-    from test.support import check_syntax_warning
-
     def setUp(self):
         self.save_path = sys.path[:]
         self.tmpdir = tempfile.mkdtemp()
@@ -119,7 +118,7 @@ class TestLiterals(unittest.TestCase):
             eval("'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -129,7 +128,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_str_raw(self):
@@ -170,7 +169,7 @@ class TestLiterals(unittest.TestCase):
             eval("b'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
@@ -180,7 +179,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        if not sys.flags.use_peg:
+        if use_old_parser():
             self.assertEqual(exc.lineno, 1)
 
     def test_eval_bytes_raw(self):

Lib/test/test_syntax.py

@@ -678,8 +678,8 @@ class SyntaxTestCase(unittest.TestCase):
     def test_assign_call(self):
         self._check_error("f() = 1", "assign")
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not produce a specialized error "
+    @support.skip_if_new_parser("Pegen does not produce a specialized error "
-                                        "message yet")
+                                "message yet")
     def test_assign_del(self):
         self._check_error("del f()", "delete")
 

Lib/test/test_sys.py

@@ -545,7 +545,7 @@ class SysModuleTest(unittest.TestCase):
     def test_sys_flags(self):
         self.assertTrue(sys.flags)
         attrs = ("debug",
-                 "inspect", "interactive", "optimize", "use_peg",
+                 "inspect", "interactive", "optimize",
                  "dont_write_bytecode", "no_user_site", "no_site",
                  "ignore_environment", "verbose", "bytes_warning", "quiet",
                  "hash_randomization", "isolated", "dev_mode", "utf8_mode")

Lib/test/test_traceback.py

@@ -656,8 +656,7 @@ class BaseExceptionReportingTests:
         self.assertIn('inner_raise() # Marker', blocks[2])
         self.check_zero_div(blocks[2])
 
-    @unittest.skipIf(sys.flags.use_peg,
+    @support.skip_if_new_parser("Pegen is arguably better here, so no need to fix this")
-                     "Pegen is arguably better here, so no need to fix this")
     def test_syntax_error_offset_at_eol(self):
         # See #10186.
         def e():

Lib/test/test_type_comments.py

@@ -1,6 +1,7 @@
 import ast
 import sys
 import unittest
+from test import support
 
 
 funcdef = """\
@@ -218,7 +219,7 @@ def favk(
 """
 
 
-@unittest.skipIf(sys.flags.use_peg, "Pegen does not support type comments yet")
+@support.skip_if_new_parser("Pegen does not support type comments yet")
 class TypeCommentTests(unittest.TestCase):
 
     lowest = 4  # Lowest minor version supported

Lib/test/test_unparse.py

@@ -328,7 +328,7 @@ class UnparseTestCase(ASTTestCase):
             ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
         )
 
-    @unittest.skipIf(sys.flags.use_peg, "Pegen does not support type annotation yet")
+    @test.support.skip_if_new_parser("Pegen does not support type annotation yet")
     def test_function_type(self):
         for function_type in (
             "() -> int",

Programs/_testembed.c

@@ -485,8 +485,8 @@ static int test_init_from_config(void)
 
     config.install_signal_handlers = 0;
 
-    putenv("PYTHONOLDPARSER=");
+    putenv("PYTHONOLDPARSER=1");
-    config.use_peg = 0;
+    config._use_peg_parser = 0;
 
     /* FIXME: test use_environment */
 
@@ -665,6 +665,7 @@ static void set_most_env_vars(void)
     putenv("PYTHONNOUSERSITE=1");
     putenv("PYTHONFAULTHANDLER=1");
     putenv("PYTHONIOENCODING=iso8859-1:replace");
+    putenv("PYTHONOLDPARSER=1");
 }
 
 

Python/bltinmodule.c

@@ -816,12 +816,12 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
     if (str == NULL)
         goto error;
 
-    int current_use_peg = PyInterpreterState_Get()->config.use_peg;
+    int current_use_peg = PyInterpreterState_Get()->config._use_peg_parser;
     if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) {
-        PyInterpreterState_Get()->config.use_peg = 0;
+        PyInterpreterState_Get()->config._use_peg_parser = 0;
     }
     result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
-    PyInterpreterState_Get()->config.use_peg = current_use_peg;
+    PyInterpreterState_Get()->config._use_peg_parser = current_use_peg;
     Py_XDECREF(source_copy);
     goto finally;
 

Python/initconfig.c

@@ -635,7 +635,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = -1;
 #endif
-    config->use_peg = 1;
+    config->_use_peg_parser = 1;
 }
 
 
@@ -793,7 +793,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
-    COPY_ATTR(use_peg);
+    COPY_ATTR(_use_peg_parser);
     COPY_ATTR(install_signal_handlers);
     COPY_ATTR(use_hash_seed);
     COPY_ATTR(hash_seed);
@@ -897,7 +897,7 @@ config_as_dict(const PyConfig *config)
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
     SET_ITEM_INT(dev_mode);
-    SET_ITEM_INT(use_peg);
+    SET_ITEM_INT(_use_peg_parser);
     SET_ITEM_INT(install_signal_handlers);
     SET_ITEM_INT(use_hash_seed);
     SET_ITEM_UINT(hash_seed);
@@ -1434,7 +1434,7 @@ config_read_complex_options(PyConfig *config)
 
     if (config_get_env(config, "PYTHONOLDPARSER")
        || config_get_xoption(config, L"oldparser")) {
-        config->use_peg = 0;
+        config->_use_peg_parser = 0;
     }
 
     PyStatus status;
@@ -2516,7 +2516,7 @@ PyConfig_Read(PyConfig *config)
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
     assert(config->dev_mode >= 0);
-    assert(config->use_peg >= 0);
+    assert(config->_use_peg_parser >= 0);
     assert(config->install_signal_handlers >= 0);
     assert(config->use_hash_seed >= 0);
     assert(config->faulthandler >= 0);

Python/pythonrun.c

@@ -185,7 +185,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
     PyArena *arena;
     const char *ps1 = "", *ps2 = "", *enc = NULL;
     int errcode = 0;
-    int use_peg = _PyInterpreterState_GET()->config.use_peg;
+    int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
     _Py_IDENTIFIER(encoding);
     _Py_IDENTIFIER(__main__);
 
@@ -1030,7 +1030,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     mod_ty mod;
     PyArena *arena;
     PyObject *filename;
-    int use_peg = _PyInterpreterState_GET()->config.use_peg;
+    int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
 
     filename = _PyUnicode_FromId(&PyId_string); /* borrowed */
     if (filename == NULL)
@@ -1061,7 +1061,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
     mod_ty mod;
     PyArena *arena = NULL;
     PyObject *filename;
-    int use_peg = _PyInterpreterState_GET()->config.use_peg;
+    int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
 
     filename = PyUnicode_DecodeFSDefault(filename_str);
     if (filename == NULL)
@@ -1222,7 +1222,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start,
 {
     PyCodeObject *co;
     mod_ty mod;
-    int use_peg = _PyInterpreterState_GET()->config.use_peg;
+    int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
     PyArena *arena = PyArena_New();
     if (arena == NULL)
         return NULL;
@@ -1329,7 +1329,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py
 {
     struct symtable *st;
     mod_ty mod;
-    int use_peg = _PyInterpreterState_GET()->config.use_peg;
+    int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
     PyArena *arena;
 
     arena = PyArena_New();

Python/sysmodule.c

@@ -2427,7 +2427,6 @@ static PyStructSequence_Field flags_fields[] = {
     {"inspect",                 "-i"},
     {"interactive",             "-i"},
     {"optimize",                "-O or -OO"},
-    {"use_peg",                 "-p old or -p new"},
     {"dont_write_bytecode",     "-B"},
     {"no_user_site",            "-s"},
     {"no_site",                 "-S"},
@@ -2448,7 +2447,7 @@ static PyStructSequence_Desc flags_desc = {
     "sys.flags",        /* name */
     flags__doc__,       /* doc */
     flags_fields,       /* fields */
-    16
+    15
 };
 
 static PyObject*
@@ -2471,7 +2470,6 @@ make_flags(PyThreadState *tstate)
     SetFlag(config->inspect);
     SetFlag(config->interactive);
     SetFlag(config->optimization_level);
-    SetFlag(config->use_peg);
     SetFlag(!config->write_bytecode);
     SetFlag(!config->user_site_directory);
     SetFlag(!config->site_import);