Compare commits
3 Commits
0b281f94b9
...
f4507231e3
Author | SHA1 | Date |
---|---|---|
Victor Stinner | f4507231e3 | |
Pablo Galindo | 3bcc4ead3f | |
Shantanu | 7865f516f3 |
|
@ -0,0 +1,51 @@
|
|||
import unittest
|
||||
from test import test_tools
|
||||
|
||||
test_tools.skip_if_missing('peg_generator')
|
||||
with test_tools.imports_under_tool('peg_generator'):
|
||||
from pegen.grammar_parser import GeneratedParser as GrammarParser
|
||||
from pegen.validator import SubRuleValidator, ValidationError
|
||||
from pegen.testutil import parse_string
|
||||
from pegen.grammar import Grammar
|
||||
|
||||
|
||||
class TestPegen(unittest.TestCase):
|
||||
def test_rule_with_no_collision(self) -> None:
|
||||
grammar_source = """
|
||||
start: bad_rule
|
||||
sum:
|
||||
| NAME '-' NAME
|
||||
| NAME '+' NAME
|
||||
"""
|
||||
grammar: Grammar = parse_string(grammar_source, GrammarParser)
|
||||
validator = SubRuleValidator(grammar)
|
||||
for rule_name, rule in grammar.rules.items():
|
||||
validator.validate_rule(rule_name, rule)
|
||||
|
||||
def test_rule_with_simple_collision(self) -> None:
|
||||
grammar_source = """
|
||||
start: bad_rule
|
||||
sum:
|
||||
| NAME '+' NAME
|
||||
| NAME '+' NAME ';'
|
||||
"""
|
||||
grammar: Grammar = parse_string(grammar_source, GrammarParser)
|
||||
validator = SubRuleValidator(grammar)
|
||||
with self.assertRaises(ValidationError):
|
||||
for rule_name, rule in grammar.rules.items():
|
||||
validator.validate_rule(rule_name, rule)
|
||||
|
||||
def test_rule_with_collision_after_some_other_rules(self) -> None:
|
||||
grammar_source = """
|
||||
start: bad_rule
|
||||
sum:
|
||||
| NAME '+' NAME
|
||||
| NAME '*' NAME ';'
|
||||
| NAME '-' NAME
|
||||
| NAME '+' NAME ';'
|
||||
"""
|
||||
grammar: Grammar = parse_string(grammar_source, GrammarParser)
|
||||
validator = SubRuleValidator(grammar)
|
||||
with self.assertRaises(ValidationError):
|
||||
for rule_name, rule in grammar.rules.items():
|
||||
validator.validate_rule(rule_name, rule)
|
|
@ -560,7 +560,7 @@ by Joongi Kim and John Belmonte.
|
|||
|
||||
..
|
||||
|
||||
.. bpo: 16936
|
||||
.. bpo: 16396
|
||||
.. date: 2020-07-08-09-45-00
|
||||
.. nonce: z8o8Pn
|
||||
.. section: Library
|
||||
|
|
|
@ -286,7 +286,9 @@ void
|
|||
_PyType_Fini(PyThreadState *tstate)
|
||||
{
|
||||
_PyType_ClearCache(&tstate->interp->type_cache);
|
||||
if (_Py_IsMainInterpreter(tstate)) {
|
||||
clear_slotdefs();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -2359,10 +2359,9 @@ _PyUnicode_FromId(_Py_Identifier *id)
|
|||
|
||||
|
||||
static void
|
||||
unicode_clear_identifiers(PyThreadState *tstate)
|
||||
unicode_clear_identifiers(struct _Py_unicode_state *state)
|
||||
{
|
||||
PyInterpreterState *interp = _PyInterpreterState_GET();
|
||||
struct _Py_unicode_ids *ids = &interp->unicode.ids;
|
||||
struct _Py_unicode_ids *ids = &state->ids;
|
||||
for (Py_ssize_t i=0; i < ids->size; i++) {
|
||||
Py_XDECREF(ids->array[i]);
|
||||
}
|
||||
|
@ -16243,7 +16242,7 @@ _PyUnicode_Fini(PyThreadState *tstate)
|
|||
|
||||
_PyUnicode_FiniEncodings(&state->fs_codec);
|
||||
|
||||
unicode_clear_identifiers(tstate);
|
||||
unicode_clear_identifiers(state);
|
||||
|
||||
for (Py_ssize_t i = 0; i < 256; i++) {
|
||||
Py_CLEAR(state->latin1[i]);
|
||||
|
|
|
@ -1573,6 +1573,7 @@ finalize_interp_types(PyThreadState *tstate)
|
|||
_PyFrame_Fini(tstate);
|
||||
_PyAsyncGen_Fini(tstate);
|
||||
_PyContext_Fini(tstate);
|
||||
_PyType_Fini(tstate);
|
||||
// Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
|
||||
// a dict internally.
|
||||
_PyUnicode_ClearInterned(tstate);
|
||||
|
@ -1751,9 +1752,6 @@ Py_FinalizeEx(void)
|
|||
/* Destroy the database used by _PyImport_{Fixup,Find}Extension */
|
||||
_PyImport_Fini();
|
||||
|
||||
/* Cleanup typeobject.c's internal caches. */
|
||||
_PyType_Fini(tstate);
|
||||
|
||||
/* unload faulthandler module */
|
||||
_PyFaulthandler_Fini();
|
||||
|
||||
|
|
|
@ -14,6 +14,7 @@ import traceback
|
|||
from typing import Tuple
|
||||
|
||||
from pegen.build import Grammar, Parser, Tokenizer, ParserGenerator
|
||||
from pegen.validator import validate_grammar
|
||||
|
||||
|
||||
def generate_c_code(
|
||||
|
@ -128,6 +129,8 @@ def main() -> None:
|
|||
grammar, parser, tokenizer, gen = args.func(args)
|
||||
t1 = time.time()
|
||||
|
||||
validate_grammar(grammar)
|
||||
|
||||
if not args.quiet:
|
||||
if args.verbose:
|
||||
print("Raw Grammar:")
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
from pegen import grammar
|
||||
from pegen.grammar import (
|
||||
Alt,
|
||||
Cut,
|
||||
Gather,
|
||||
GrammarVisitor,
|
||||
Group,
|
||||
Lookahead,
|
||||
NamedItem,
|
||||
NameLeaf,
|
||||
NegativeLookahead,
|
||||
Opt,
|
||||
PositiveLookahead,
|
||||
Repeat0,
|
||||
Repeat1,
|
||||
Rhs,
|
||||
Rule,
|
||||
StringLeaf,
|
||||
)
|
||||
|
||||
class ValidationError(Exception):
|
||||
pass
|
||||
|
||||
class GrammarValidator(GrammarVisitor):
|
||||
def __init__(self, grammar: grammar.Grammar):
|
||||
self.grammar = grammar
|
||||
self.rulename = None
|
||||
|
||||
def validate_rule(self, rulename: str, node: Rule):
|
||||
self.rulename = rulename
|
||||
self.visit(node)
|
||||
self.rulename = None
|
||||
|
||||
|
||||
class SubRuleValidator(GrammarValidator):
|
||||
def visit_Rhs(self, node: Rule):
|
||||
for index, alt in enumerate(node.alts):
|
||||
alts_to_consider = node.alts[index+1:]
|
||||
for other_alt in alts_to_consider:
|
||||
self.check_intersection(alt, other_alt)
|
||||
|
||||
def check_intersection(self, first_alt: Alt, second_alt: Alt) -> bool:
|
||||
if str(second_alt).startswith(str(first_alt)):
|
||||
raise ValidationError(
|
||||
f"In {self.rulename} there is an alternative that will "
|
||||
f"never be visited:\n{second_alt}")
|
||||
|
||||
def validate_grammar(the_grammar: grammar.Grammar):
|
||||
for validator_cls in GrammarValidator.__subclasses__():
|
||||
validator = validator_cls(the_grammar)
|
||||
for rule_name, rule in the_grammar.rules.items():
|
||||
validator.validate_rule(rule_name, rule)
|
Loading…
Reference in New Issue