bpo-42745: finalize_interp_types() calls _PyType_Fini() (GH-23953)

Call _PyType_Fini() in subinterpreters.

Fix reference leaks in subinterpreters.

Lib/test/test_peg_generator/test_grammar_validator.py

@@ -0,0 +1,51 @@
+import unittest
+from test import test_tools
+
+test_tools.skip_if_missing('peg_generator')
+with test_tools.imports_under_tool('peg_generator'):
+    from pegen.grammar_parser import GeneratedParser as GrammarParser
+    from pegen.validator import SubRuleValidator, ValidationError
+    from pegen.testutil import parse_string
+    from pegen.grammar import Grammar
+
+
+class TestPegen(unittest.TestCase):
+    def test_rule_with_no_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '-' NAME
+            | NAME '+' NAME
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        for rule_name, rule in grammar.rules.items():
+            validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_simple_collision(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)
+
+    def test_rule_with_collision_after_some_other_rules(self) -> None:
+        grammar_source = """
+        start: bad_rule
+        sum:
+            | NAME '+' NAME
+            | NAME '*' NAME ';'
+            | NAME '-' NAME
+            | NAME '+' NAME ';'
+        """
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        validator = SubRuleValidator(grammar)
+        with self.assertRaises(ValidationError):
+            for rule_name, rule in grammar.rules.items():
+                validator.validate_rule(rule_name, rule)

Misc/NEWS.d/3.10.0a2.rst

@@ -560,7 +560,7 @@ by Joongi Kim and John Belmonte.
 
 ..
 
-.. bpo: 16936
+.. bpo: 16396
 .. date: 2020-07-08-09-45-00
 .. nonce: z8o8Pn
 .. section: Library

Objects/typeobject.c

@@ -286,7 +286,9 @@ void
 _PyType_Fini(PyThreadState *tstate)
 {
     _PyType_ClearCache(&tstate->interp->type_cache);
-    clear_slotdefs();
+    if (_Py_IsMainInterpreter(tstate)) {
+        clear_slotdefs();
+    }
 }
 
 

Objects/unicodeobject.c

@@ -2359,10 +2359,9 @@ _PyUnicode_FromId(_Py_Identifier *id)
 
 
 static void
-unicode_clear_identifiers(PyThreadState *tstate)
+unicode_clear_identifiers(struct _Py_unicode_state *state)
 {
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _Py_unicode_ids *ids = &interp->unicode.ids;
+    struct _Py_unicode_ids *ids = &state->ids;
     for (Py_ssize_t i=0; i < ids->size; i++) {
         Py_XDECREF(ids->array[i]);
     }
@@ -16243,7 +16242,7 @@ _PyUnicode_Fini(PyThreadState *tstate)
 
     _PyUnicode_FiniEncodings(&state->fs_codec);
 
-    unicode_clear_identifiers(tstate);
+    unicode_clear_identifiers(state);
 
     for (Py_ssize_t i = 0; i < 256; i++) {
         Py_CLEAR(state->latin1[i]);

Python/pylifecycle.c

@@ -1573,6 +1573,7 @@ finalize_interp_types(PyThreadState *tstate)
     _PyFrame_Fini(tstate);
     _PyAsyncGen_Fini(tstate);
     _PyContext_Fini(tstate);
+    _PyType_Fini(tstate);
     // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
     // a dict internally.
     _PyUnicode_ClearInterned(tstate);
@@ -1751,9 +1752,6 @@ Py_FinalizeEx(void)
     /* Destroy the database used by _PyImport_{Fixup,Find}Extension */
     _PyImport_Fini();
 
-    /* Cleanup typeobject.c's internal caches. */
-    _PyType_Fini(tstate);
-
     /* unload faulthandler module */
     _PyFaulthandler_Fini();
 

Tools/peg_generator/pegen/__main__.py

@@ -14,6 +14,7 @@ import traceback
 from typing import Tuple
 
 from pegen.build import Grammar, Parser, Tokenizer, ParserGenerator
+from pegen.validator import validate_grammar
 
 
 def generate_c_code(
@@ -128,6 +129,8 @@ def main() -> None:
     grammar, parser, tokenizer, gen = args.func(args)
     t1 = time.time()
 
+    validate_grammar(grammar)
+
     if not args.quiet:
         if args.verbose:
             print("Raw Grammar:")

Tools/peg_generator/pegen/validator.py

@@ -0,0 +1,52 @@
+from pegen import grammar
+from pegen.grammar import (
+    Alt,
+    Cut,
+    Gather,
+    GrammarVisitor,
+    Group,
+    Lookahead,
+    NamedItem,
+    NameLeaf,
+    NegativeLookahead,
+    Opt,
+    PositiveLookahead,
+    Repeat0,
+    Repeat1,
+    Rhs,
+    Rule,
+    StringLeaf,
+)
+
+class ValidationError(Exception):
+    pass
+
+class GrammarValidator(GrammarVisitor):
+    def __init__(self, grammar: grammar.Grammar):
+        self.grammar = grammar
+        self.rulename = None
+
+    def validate_rule(self, rulename: str, node: Rule):
+        self.rulename = rulename
+        self.visit(node)
+        self.rulename = None
+
+
+class SubRuleValidator(GrammarValidator):
+    def visit_Rhs(self, node: Rule):
+        for index, alt in enumerate(node.alts):
+            alts_to_consider = node.alts[index+1:]
+            for other_alt in alts_to_consider:
+                self.check_intersection(alt, other_alt)
+
+    def check_intersection(self, first_alt: Alt, second_alt: Alt) -> bool:
+        if str(second_alt).startswith(str(first_alt)):
+            raise ValidationError(
+                    f"In {self.rulename} there is an alternative that will "
+                    f"never be visited:\n{second_alt}")
+
+def validate_grammar(the_grammar: grammar.Grammar):
+    for validator_cls in GrammarValidator.__subclasses__():
+        validator = validator_cls(the_grammar)
+        for rule_name, rule in the_grammar.rules.items():
+            validator.validate_rule(rule_name, rule)