"""Python parser generator This parser generator transforms a Python grammar file into parsing tables that can be consumed by Python's LL(1) parser written in C. Concepts -------- * An LL(1) parser (Left-to-right, Leftmost derivation, 1 token-lookahead) is a top-down parser for a subset of context-free languages. It parses the input from Left to right, performing Leftmost derivation of the sentence, and can only use 1 token of lookahead when parsing a sentence. * A parsing table is a collection of data that a generic implementation of the LL(1) parser consumes to know how to parse a given context-free grammar. In this case the collection of data involves Deterministic Finite Automatons, calculated first sets, keywords and transition labels. * A grammar is defined by production rules (or just 'productions') that specify which symbols may replace which other symbols; these rules may be used to generate strings, or to parse them. Each such rule has a head, or left-hand side, which consists of the string that may be replaced, and a body, or right-hand side, which consists of a string that may replace it. In the Python grammar, rules are written in the form rule_name: rule_description; meaning the rule 'a: b' specifies that a can be replaced by b. A context-free grammar is a grammar in which the left-hand side of each production rule consists of only a single nonterminal symbol. Context-free grammars can always be recognized by a Non-Deterministic Automatons. * Terminal symbols are literal symbols which may appear in the outputs of the production rules of the grammar and which cannot be changed using the rules of the grammar. Applying the rules recursively to a source string of symbols will usually terminate in a final output string consisting only of terminal symbols. * Nonterminal symbols are those symbols which can be replaced. The grammar includes a start symbol a designated member of the set of nonterminals from which all the strings in the language may be derived by successive applications of the production rules. * The language defined by the grammar is defined as the set of terminal strings that can be derived using the production rules. * The first sets of a rule (FIRST(rule)) are defined to be the set of terminals that can appear in the first position of any string derived from the rule. This is useful for LL(1) parsers as the parser is only allowed to look at the next token in the input to know which rule needs to parse. For example, given this grammar: start: '(' A | B ')' A: 'a' '<' B: 'b' '<' and the input '(b<)' the parser can only look at 'b' to know if it needs to parse A o B. Because FIRST(A) = {'a'} and FIRST(B) = {'b'} it knows that needs to continue parsing rule B because only that rule can start with 'b'. Description ----------- The input for the parser generator is a grammar in extended BNF form (using * for repetition, + for at-least-once repetition, [] for optional parts, | for alternatives and () for grouping). Each rule in the grammar file is considered as a regular expression in its own right. It is turned into a Non-deterministic Finite Automaton (NFA), which is then turned into a Deterministic Finite Automaton (DFA), which is then optimized to reduce the number of states. See [Aho&Ullman 77] chapter 3, or similar compiler books (this technique is more often used for lexical analyzers). The DFA's are used by the parser as parsing tables in a special way that's probably unique. Before they are usable, the FIRST sets of all non-terminals are computed so the LL(1) parser consuming the parsing tables can distinguish between different transitions. Reference --------- [Aho&Ullman 77] Aho&Ullman, Principles of Compiler Design, Addison-Wesley 1977 (first edition) """ from ast import literal_eval import collections from . import grammar, token from .automata import DFA from .metaparser import GrammarParser import enum class LabelType(enum.Enum): NONTERMINAL = 0 NAMED_TOKEN = 1 KEYWORD = 2 OPERATOR = 3 NONE = 4 class Label(str): def __init__(self, value): self.type = self._get_type() def _get_type(self): if self[0].isalpha(): if self.upper() == self: # NAMED tokens (ASYNC, NAME...) are all uppercase by convention return LabelType.NAMED_TOKEN else: # If is not uppercase it must be a non terminal. return LabelType.NONTERMINAL else: # Keywords and operators are wrapped in quotes assert self[0] == self[-1] in ('"', "'"), self value = literal_eval(self) if value[0].isalpha(): return LabelType.KEYWORD else: return LabelType.OPERATOR def __repr__(self): return "{}({})".format(self.type, super().__repr__()) class ParserGenerator(object): def __init__(self, grammar_file, token_file, verbose=False): with open(grammar_file) as f: self.grammar = f.read() with open(token_file) as tok_file: token_lines = tok_file.readlines() self.tokens = dict(token.generate_tokens(token_lines)) self.opmap = dict(token.generate_opmap(token_lines)) # Manually add <> so it does not collide with != self.opmap["<>"] = "NOTEQUAL" self.verbose = verbose self.filename = grammar_file self.dfas, self.startsymbol = self.create_dfas() self.first = {} # map from symbol name to set of tokens self.calculate_first_sets() def create_dfas(self): rule_to_dfas = collections.OrderedDict() start_nonterminal = None for nfa in GrammarParser(self.grammar).parse(): if self.verbose: print("Dump of NFA for", nfa.name) nfa.dump() dfa = DFA.from_nfa(nfa) if self.verbose: print("Dump of DFA for", dfa.name) dfa.dump() dfa.simplify() rule_to_dfas[dfa.name] = dfa if start_nonterminal is None: start_nonterminal = dfa.name return rule_to_dfas, start_nonterminal def make_grammar(self): c = grammar.Grammar() c.all_labels = set() names = list(self.dfas.keys()) names.remove(self.startsymbol) names.insert(0, self.startsymbol) for name in names: i = 256 + len(c.symbol2number) c.symbol2number[Label(name)] = i c.number2symbol[i] = Label(name) c.all_labels.add(name) for name in names: self.make_label(c, name) dfa = self.dfas[name] states = [] for state in dfa: arcs = [] for label, next in sorted(state.arcs.items()): c.all_labels.add(label) arcs.append((self.make_label(c, label), dfa.states.index(next))) if state.is_final: arcs.append((0, dfa.states.index(state))) states.append(arcs) c.states.append(states) c.dfas[c.symbol2number[name]] = (states, self.make_first_sets(c, name)) c.start = c.symbol2number[self.startsymbol] if self.verbose: print("") print("Grammar summary") print("===============") print("- {n_labels} labels".format(n_labels=len(c.labels))) print("- {n_dfas} dfas".format(n_dfas=len(c.dfas))) print("- {n_tokens} tokens".format(n_tokens=len(c.tokens))) print("- {n_keywords} keywords".format(n_keywords=len(c.keywords))) print( "- Start symbol: {start_symbol}".format( start_symbol=c.number2symbol[c.start] ) ) return c def make_first_sets(self, c, name): rawfirst = self.first[name] first = set() for label in sorted(rawfirst): ilabel = self.make_label(c, label) ##assert ilabel not in first # XXX failed on <> ... != first.add(ilabel) return first def make_label(self, c, label): label = Label(label) ilabel = len(c.labels) if label.type == LabelType.NONTERMINAL: if label in c.symbol2label: return c.symbol2label[label] else: c.labels.append((c.symbol2number[label], None)) c.symbol2label[label] = ilabel return ilabel elif label.type == LabelType.NAMED_TOKEN: # A named token (NAME, NUMBER, STRING) itoken = self.tokens.get(label, None) assert isinstance(itoken, int), label assert itoken in self.tokens.values(), label if itoken in c.tokens: return c.tokens[itoken] else: c.labels.append((itoken, None)) c.tokens[itoken] = ilabel return ilabel elif label.type == LabelType.KEYWORD: # A keyword value = literal_eval(label) if value in c.keywords: return c.keywords[value] else: c.labels.append((self.tokens["NAME"], value)) c.keywords[value] = ilabel return ilabel elif label.type == LabelType.OPERATOR: # An operator (any non-numeric token) value = literal_eval(label) tok_name = self.opmap[value] # Fails if unknown token itoken = self.tokens[tok_name] if itoken in c.tokens: return c.tokens[itoken] else: c.labels.append((itoken, None)) c.tokens[itoken] = ilabel return ilabel else: raise ValueError("Cannot categorize label {}".format(label)) def calculate_first_sets(self): names = list(self.dfas.keys()) for name in names: if name not in self.first: self.calculate_first_sets_for_rule(name) if self.verbose: print("First set for {dfa_name}".format(dfa_name=name)) for item in self.first[name]: print(" - {terminal}".format(terminal=item)) def calculate_first_sets_for_rule(self, name): dfa = self.dfas[name] self.first[name] = None # dummy to detect left recursion state = dfa.states[0] totalset = set() overlapcheck = {} for label, next in state.arcs.items(): if label in self.dfas: if label in self.first: fset = self.first[label] if fset is None: raise ValueError("recursion for rule %r" % name) else: self.calculate_first_sets_for_rule(label) fset = self.first[label] totalset.update(fset) overlapcheck[label] = fset else: totalset.add(label) overlapcheck[label] = {label} inverse = {} for label, itsfirst in overlapcheck.items(): for symbol in itsfirst: if symbol in inverse: raise ValueError( "rule %s is ambiguous; %s is in the" " first sets of %s as well as %s" % (name, symbol, label, inverse[symbol]) ) inverse[symbol] = label self.first[name] = totalset