traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-40334: Rename PyConfig.use_peg to _use_peg_parser (GH-19670) 

* Rename PyConfig.use_peg to _use_peg_parser
* Document PyConfig._use_peg_parser and mark it a deprecated
* Mark -X oldparser option and PYTHONOLDPARSER env var as deprecated
  in the documentation.
* Add use_old_parser() and skip_if_new_parser() to test.support
* Remove sys.flags.use_peg: use_old_parser() uses
  _testinternalcapi.get_configs() instead.
* Enhance test_embed tests
* subprocess._args_from_interpreter_flags() copies -X oldparser
This commit is contained in:
Victor Stinner 2020-04-23 03:03:24 +02:00 committed by GitHub
parent a25f3c4c8f
commit 1def7754b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 83 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Doc/c-api/init_config.rst
View File

@ -686,6 +686,16 @@ PyConfig
:data:`sys._xoptions`.
.. c:member:: int _use_peg_parser
Enable PEG parser? Default: 1.
Set to 0 by :option:`-X oldparser <-X>` and :envvar:`PYTHONOLDPARSER`.
See also :pep:`617`.
.. deprecated-removed:: 3.9 3.10
If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same
way the regular Python parses command line arguments, and Python
arguments are stripped from ``argv``: see :ref:`Command Line Arguments

12
Doc/using/cmdline.rst
View File

@ -427,7 +427,7 @@ Miscellaneous options
* ``-X faulthandler`` to enable :mod:`faulthandler`;
* ``-X oldparser``: enable the traditional LL(1) parser. See also
:envvar:`PYTHONOLDPARSER`.
:envvar:`PYTHONOLDPARSER` and :pep:`617`.
* ``-X showrefcount`` to output the total reference count and number of used
memory blocks when the program finishes or after each statement in the
interactive interpreter. This only works on debug builds.
@ -480,6 +480,9 @@ Miscellaneous options
The ``-X showalloccount`` option has been removed.
.. deprecated-removed:: 3.9 3.10
The ``-X oldparser`` option.
Options you shouldn't use
~~~~~~~~~~~~~~~~~~~~~~~~~
@ -578,8 +581,11 @@ conflict.
.. envvar:: PYTHONOLDPARSER
If this is set it is equivalent to specifying the :option:`-X`
``oldparser`` option.
If this is set to a non-empty string, enable the traditional LL(1) parser.
See also the :option:`-X` ``oldparser`` option and :pep:`617`.
.. deprecated-removed:: 3.9 3.10
.. envvar:: PYTHONINSPECT

2
Include/cpython/initconfig.h
View File

@ -149,7 +149,7 @@ typedef struct {
/* Enable PEG parser?
1 by default, set to 0 by -X oldparser and PYTHONOLDPARSER */
int use_peg;
int _use_peg_parser;
/* Enable tracemalloc?
Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */

2
Lib/subprocess.py
View File

@ -326,7 +326,7 @@ def _args_from_interpreter_flags():
if dev_mode:
args.extend(('-X', 'dev'))
for opt in ('faulthandler', 'tracemalloc', 'importtime',
'showrefcount', 'utf8'):
'showrefcount', 'utf8', 'oldparser'):
if opt in xoptions:
value = xoptions[opt]
if value is True:

10
Lib/test/support/__init__.py
View File

@ -3454,3 +3454,13 @@ def wait_process(pid, *, exitcode, timeout=None):
# sanity check: it should not fail in practice
if pid2 != pid:
raise AssertionError(f"pid {pid2} != pid {pid}")
def use_old_parser():
import _testinternalcapi
config = _testinternalcapi.get_configs()
return (config['config']['_use_peg_parser'] == 0)
def skip_if_new_parser(msg):
return unittest.skipIf(not use_old_parser(), msg)

10
Lib/test/test_codeop.py
View File

@ -4,12 +4,12 @@
"""
import sys
import unittest
from test.support import is_jython
from test import support
from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
import io
if is_jython:
if support.is_jython:
def unify_callables(d):
for n,v in d.items():
@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase):
def assertValid(self, str, symbol='single'):
'''succeed iff str is a valid piece of code'''
if is_jython:
if support.is_jython:
code = compile_command(str, "<input>", symbol)
self.assertTrue(code)
if symbol == "single":
@ -60,7 +60,7 @@ class CodeopTests(unittest.TestCase):
av = self.assertValid
# special case
if not is_jython:
if not support.is_jython:
self.assertEqual(compile_command(""),
compile("pass", "<input>", 'single',
PyCF_DONT_IMPLY_DEDENT))
@ -122,7 +122,7 @@ class CodeopTests(unittest.TestCase):
av("def f():\n pass\n#foo\n")
av("@a.b.c\ndef f():\n pass\n")
@unittest.skipIf(sys.flags.use_peg, "Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
@support.skip_if_new_parser("Pegen does not support PyCF_DONT_INPLY_DEDENT yet")
def test_incomplete(self):
ai = self.assertIncomplete

2
Lib/test/test_compile.py
View File

@ -501,7 +501,7 @@ if 1:
self.compile_single("if x:\n f(x)\nelse:\n g(x)")
self.compile_single("class T:\n pass")
@unittest.skipIf(sys.flags.use_peg, 'Pegen does not disallow multiline single stmts')
@support.skip_if_new_parser('Pegen does not disallow multiline single stmts')
def test_bad_single_statement(self):
self.assertInvalidSingle('1\n2')
self.assertInvalidSingle('def f(): pass')

6
Lib/test/test_embed.py
View File

@ -347,7 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'isolated': 0,
'use_environment': 1,
'dev_mode': 0,
'use_peg': 1,
'_use_peg_parser': 1,
'install_signal_handlers': 1,
'use_hash_seed': 0,
@ -729,7 +729,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'import_time': 1,
'show_ref_count': 1,
'malloc_stats': 1,
'use_peg': 0,
'_use_peg_parser': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
@ -792,6 +792,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'user_site_directory': 0,
'faulthandler': 1,
'warnoptions': ['EnvVar'],
'_use_peg_parser': 0,
}
self.check_all_configs("test_init_compat_env", config, preconfig,
api=API_COMPAT)
@ -819,6 +820,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'user_site_directory': 0,
'faulthandler': 1,
'warnoptions': ['EnvVar'],
'_use_peg_parser': 0,
}
self.check_all_configs("test_init_python_env", config, preconfig,
api=API_PYTHON)

2
Lib/test/test_eof.py
View File

@ -26,7 +26,7 @@ class EOFTestCase(unittest.TestCase):
else:
raise support.TestFailed
@unittest.skipIf(sys.flags.use_peg, "TODO for PEG -- fails with new parser")
@support.skip_if_new_parser("TODO for PEG -- fails with new parser")
def test_line_continuation_EOF(self):
"""A continuation at the end of input must be an error; bpo2180."""
expect = 'unexpected EOF while parsing (<string>, line 1)'

2
Lib/test/test_exceptions.py
View File

@ -178,7 +178,7 @@ class ExceptionTests(unittest.TestCase):
s = '''if True:\n print()\n\texec "mixed tabs and spaces"'''
ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError)
@unittest.skipIf(sys.flags.use_peg, "Pegen column offsets might be different")
@support.skip_if_new_parser("Pegen column offsets might be different")
def testSyntaxErrorOffset(self):
def check(src, lineno, offset, encoding='utf-8'):
with self.assertRaises(SyntaxError) as cm:

3
Lib/test/test_flufl.py
View File

@ -1,9 +1,10 @@
import __future__
import unittest
import sys
from test import support
@unittest.skipIf(sys.flags.use_peg, "Not supported by pegen yet")
@support.skip_if_new_parser("Not supported by pegen yet")
class FLUFLTests(unittest.TestCase):
def test_barry_as_bdfl(self):

3
Lib/test/test_fstring.py
View File

@ -12,6 +12,7 @@ import types
import decimal
import sys
import unittest
from test import support
a_global = 'global variable'
@ -206,7 +207,7 @@ f'{a * f"-{x()}-"}'"""
call = binop.right.values[1].value
self.assertEqual(type(call), ast.Call)
self.assertEqual(call.lineno, 3)
if not sys.flags.use_peg:
if support.use_old_parser():
self.assertEqual(call.col_offset, 11)
def test_ast_line_numbers_duplicate_expression(self):

2
Lib/test/test_parser.py
View File

@ -900,7 +900,7 @@ class ParserStackLimitTestCase(unittest.TestCase):
st = parser.expr(e)
st.compile()
@unittest.skipIf(sys.flags.use_peg, "Pegen does not trigger memory error with this many parenthesis")
@support.skip_if_new_parser("Pegen does not trigger memory error with this many parenthesis")
def test_trigger_memory_error(self):
e = self._nested_expression(100)
rc, out, err = assert_python_failure('-Xoldparser', '-c', e)

3
Lib/test/test_peg_parser.py
View File

@ -6,6 +6,7 @@ import unittest
from pathlib import PurePath
from typing import Any, Union, Iterable, Tuple
from textwrap import dedent
from test import support
TEST_CASES = [
@ -720,7 +721,7 @@ class ASTGenerationTest(unittest.TestCase):
f"Actual error message does not match expexted for {source}"
)
@unittest.skipIf(sys.flags.use_peg, "This tests nothing for now, since compile uses pegen as well")
@support.skip_if_new_parser("This tests nothing for now, since compile uses pegen as well")
@unittest.expectedFailure
def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
for source in GOOD_BUT_FAIL_SOURCES:

8
Lib/test/test_positional_only_arg.py
View File

@ -5,7 +5,7 @@ import pickle
import unittest
import sys
from test.support import check_syntax_error
from test.support import check_syntax_error, use_old_parser
def global_pos_only_f(a, b, /):
@ -24,7 +24,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
compile(codestr + "\n", "<test>", "single")
def test_invalid_syntax_errors(self):
if not sys.flags.use_peg:
if use_old_parser():
check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument")
@ -47,7 +47,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
check_syntax_error(self, "def f(a, *, c, /, d, e): pass")
def test_invalid_syntax_errors_async(self):
if not sys.flags.use_peg:
if use_old_parser():
check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument")
check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument")
@ -236,7 +236,7 @@ class PositionalOnlyTestCase(unittest.TestCase):
self.assertEqual(x(1, 2), 3)
def test_invalid_syntax_lambda(self):
if not sys.flags.use_peg:
if use_old_parser():
check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument")
check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument")
check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument")

11
Lib/test/test_string_literals.py
View File

@ -33,6 +33,7 @@ import shutil
import tempfile
import unittest
import warnings
from test.support import check_syntax_warning, use_old_parser
TEMPLATE = r"""# coding: %s
@ -63,8 +64,6 @@ def byte(i):
class TestLiterals(unittest.TestCase):
from test.support import check_syntax_warning
def setUp(self):
self.save_path = sys.path[:]
self.tmpdir = tempfile.mkdtemp()
@ -119,7 +118,7 @@ class TestLiterals(unittest.TestCase):
eval("'''\n\\z'''")
self.assertEqual(len(w), 1)
self.assertEqual(w[0].filename, '<string>')
if not sys.flags.use_peg:
if use_old_parser():
self.assertEqual(w[0].lineno, 1)
with warnings.catch_warnings(record=True) as w:
@ -129,7 +128,7 @@ class TestLiterals(unittest.TestCase):
exc = cm.exception
self.assertEqual(w, [])
self.assertEqual(exc.filename, '<string>')
if not sys.flags.use_peg:
if use_old_parser():
self.assertEqual(exc.lineno, 1)
def test_eval_str_raw(self):
@ -170,7 +169,7 @@ class TestLiterals(unittest.TestCase):
eval("b'''\n\\z'''")
self.assertEqual(len(w), 1)
self.assertEqual(w[0].filename, '<string>')
if not sys.flags.use_peg:
if use_old_parser():
self.assertEqual(w[0].lineno, 1)
with warnings.catch_warnings(record=True) as w:
@ -180,7 +179,7 @@ class TestLiterals(unittest.TestCase):
exc = cm.exception
self.assertEqual(w, [])
self.assertEqual(exc.filename, '<string>')
if not sys.flags.use_peg:
if use_old_parser():
self.assertEqual(exc.lineno, 1)
def test_eval_bytes_raw(self):

2
Lib/test/test_syntax.py
View File

@ -678,7 +678,7 @@ class SyntaxTestCase(unittest.TestCase):
def test_assign_call(self):
self._check_error("f() = 1", "assign")
@unittest.skipIf(sys.flags.use_peg, "Pegen does not produce a specialized error "
@support.skip_if_new_parser("Pegen does not produce a specialized error "
"message yet")
def test_assign_del(self):
self._check_error("del f()", "delete")

2
Lib/test/test_sys.py
View File

@ -545,7 +545,7 @@ class SysModuleTest(unittest.TestCase):
def test_sys_flags(self):
self.assertTrue(sys.flags)
attrs = ("debug",
"inspect", "interactive", "optimize", "use_peg",
"inspect", "interactive", "optimize",
"dont_write_bytecode", "no_user_site", "no_site",
"ignore_environment", "verbose", "bytes_warning", "quiet",
"hash_randomization", "isolated", "dev_mode", "utf8_mode")

3
Lib/test/test_traceback.py
View File

@ -656,8 +656,7 @@ class BaseExceptionReportingTests:
self.assertIn('inner_raise() # Marker', blocks[2])
self.check_zero_div(blocks[2])
@unittest.skipIf(sys.flags.use_peg,
"Pegen is arguably better here, so no need to fix this")
@support.skip_if_new_parser("Pegen is arguably better here, so no need to fix this")
def test_syntax_error_offset_at_eol(self):
# See #10186.
def e():

3
Lib/test/test_type_comments.py
View File

@ -1,6 +1,7 @@
import ast
import sys
import unittest
from test import support
funcdef = """\
@ -218,7 +219,7 @@ def favk(
"""
@unittest.skipIf(sys.flags.use_peg, "Pegen does not support type comments yet")
@support.skip_if_new_parser("Pegen does not support type comments yet")
class TypeCommentTests(unittest.TestCase):
lowest = 4 # Lowest minor version supported

2
Lib/test/test_unparse.py
View File

@ -328,7 +328,7 @@ class UnparseTestCase(ASTTestCase):
ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
)
@unittest.skipIf(sys.flags.use_peg, "Pegen does not support type annotation yet")
@test.support.skip_if_new_parser("Pegen does not support type annotation yet")
def test_function_type(self):
for function_type in (
"() -> int",

5
Programs/_testembed.c
View File

@ -485,8 +485,8 @@ static int test_init_from_config(void)
config.install_signal_handlers = 0;
putenv("PYTHONOLDPARSER=");
config.use_peg = 0;
putenv("PYTHONOLDPARSER=1");
config._use_peg_parser = 0;
/* FIXME: test use_environment */
@ -665,6 +665,7 @@ static void set_most_env_vars(void)
putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1");
putenv("PYTHONIOENCODING=iso8859-1:replace");
putenv("PYTHONOLDPARSER=1");
}

6
Python/bltinmodule.c
View File

@ -816,12 +816,12 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
if (str == NULL)
goto error;
int current_use_peg = PyInterpreterState_Get()->config.use_peg;
int current_use_peg = PyInterpreterState_Get()->config._use_peg_parser;
if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) {
PyInterpreterState_Get()->config.use_peg = 0;
PyInterpreterState_Get()->config._use_peg_parser = 0;
}
result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
PyInterpreterState_Get()->config.use_peg = current_use_peg;
PyInterpreterState_Get()->config._use_peg_parser = current_use_peg;
Py_XDECREF(source_copy);
goto finally;

10
Python/initconfig.c
View File

@ -635,7 +635,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
#ifdef MS_WINDOWS
config->legacy_windows_stdio = -1;
#endif
config->use_peg = 1;
config->_use_peg_parser = 1;
}
@ -793,7 +793,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
COPY_ATTR(isolated);
COPY_ATTR(use_environment);
COPY_ATTR(dev_mode);
COPY_ATTR(use_peg);
COPY_ATTR(_use_peg_parser);
COPY_ATTR(install_signal_handlers);
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
@ -897,7 +897,7 @@ config_as_dict(const PyConfig *config)
SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
SET_ITEM_INT(dev_mode);
SET_ITEM_INT(use_peg);
SET_ITEM_INT(_use_peg_parser);
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
SET_ITEM_UINT(hash_seed);
@ -1434,7 +1434,7 @@ config_read_complex_options(PyConfig *config)
if (config_get_env(config, "PYTHONOLDPARSER")
|| config_get_xoption(config, L"oldparser")) {
config->use_peg = 0;
config->_use_peg_parser = 0;
}
PyStatus status;
@ -2516,7 +2516,7 @@ PyConfig_Read(PyConfig *config)
assert(config->isolated >= 0);
assert(config->use_environment >= 0);
assert(config->dev_mode >= 0);
assert(config->use_peg >= 0);
assert(config->_use_peg_parser >= 0);
assert(config->install_signal_handlers >= 0);
assert(config->use_hash_seed >= 0);
assert(config->faulthandler >= 0);

10
Python/pythonrun.c
View File

@ -185,7 +185,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
PyArena *arena;
const char *ps1 = "", *ps2 = "", *enc = NULL;
int errcode = 0;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
_Py_IDENTIFIER(encoding);
_Py_IDENTIFIER(__main__);
@ -1030,7 +1030,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
mod_ty mod;
PyArena *arena;
PyObject *filename;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
filename = _PyUnicode_FromId(&PyId_string); /* borrowed */
if (filename == NULL)
@ -1061,7 +1061,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
mod_ty mod;
PyArena *arena = NULL;
PyObject *filename;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
filename = PyUnicode_DecodeFSDefault(filename_str);
if (filename == NULL)
@ -1222,7 +1222,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start,
{
PyCodeObject *co;
mod_ty mod;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
PyArena *arena = PyArena_New();
if (arena == NULL)
return NULL;
@ -1329,7 +1329,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py
{
struct symtable *st;
mod_ty mod;
int use_peg = _PyInterpreterState_GET()->config.use_peg;
int use_peg = _PyInterpreterState_GET()->config._use_peg_parser;
PyArena *arena;
arena = PyArena_New();

4
Python/sysmodule.c
View File

@ -2427,7 +2427,6 @@ static PyStructSequence_Field flags_fields[] = {
{"inspect", "-i"},
{"interactive", "-i"},
{"optimize", "-O or -OO"},
{"use_peg", "-p old or -p new"},
{"dont_write_bytecode", "-B"},
{"no_user_site", "-s"},
{"no_site", "-S"},
@ -2448,7 +2447,7 @@ static PyStructSequence_Desc flags_desc = {
"sys.flags", /* name */
flags__doc__, /* doc */
flags_fields, /* fields */
16
15
};
static PyObject*
@ -2471,7 +2470,6 @@ make_flags(PyThreadState *tstate)
SetFlag(config->inspect);
SetFlag(config->interactive);
SetFlag(config->optimization_level);
SetFlag(config->use_peg);
SetFlag(!config->write_bytecode);
SetFlag(!config->user_site_directory);
SetFlag(!config->site_import);