X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/2082a325fdd14f0aabd88f7f12a20f9fb085c538..c83ad6c077e7bb281cfd3fbdd89bbeb4c980e563:/src/blib2to3/pgen2/pgen.py diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 13ec51d..3ece9bb 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -1,27 +1,22 @@ # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. -# Pgen imports -from . import grammar, token, tokenize - +import os from typing import ( + IO, Any, Dict, - IO, - Iterable, Iterator, List, + NoReturn, Optional, - Text, + Sequence, Tuple, Union, - Sequence, - NoReturn, ) -from blib2to3.pgen2 import grammar -from blib2to3.pgen2.tokenize import GoodTokenInfo -import os +from blib2to3.pgen2 import grammar, token, tokenize +from blib2to3.pgen2.tokenize import GoodTokenInfo Path = Union[str, "os.PathLike[str]"] @@ -30,17 +25,16 @@ class PgenGrammar(grammar.Grammar): pass -class ParserGenerator(object): - +class ParserGenerator: filename: Path - stream: IO[Text] + stream: IO[str] generator: Iterator[GoodTokenInfo] - first: Dict[Text, Optional[Dict[Text, int]]] + first: Dict[str, Optional[Dict[str, int]]] - def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: + def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: close_stream = None if stream is None: - stream = open(filename) + stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename self.stream = stream @@ -77,7 +71,7 @@ class ParserGenerator(object): c.start = c.symbol2number[self.startsymbol] return c - def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: + def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]: rawfirst = self.first[name] assert rawfirst is not None first = {} @@ -87,7 +81,7 @@ class ParserGenerator(object): first[ilabel] = 1 return first - def make_label(self, c: PgenGrammar, label: Text) -> int: + def make_label(self, c: PgenGrammar, label: str) -> int: # XXX Maybe this should be a method on a subclass of converter? ilabel = len(c.labels) if label[0].isalpha(): @@ -116,12 +110,17 @@ class ParserGenerator(object): assert label[0] in ('"', "'"), label value = eval(label) if value[0].isalpha(): + if label[0] == '"': + keywords = c.soft_keywords + else: + keywords = c.keywords + # A keyword - if value in c.keywords: - return c.keywords[value] + if value in keywords: + return keywords[value] else: c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel + keywords[value] = ilabel return ilabel else: # An operator (any non-numeric token) @@ -141,13 +140,13 @@ class ParserGenerator(object): self.calcfirst(name) # print name, self.first[name].keys() - def calcfirst(self, name: Text) -> None: + def calcfirst(self, name: str) -> None: dfa = self.dfas[name] self.first[name] = None # dummy to detect left recursion state = dfa[0] totalset: Dict[str, int] = {} overlapcheck = {} - for label, next in state.arcs.items(): + for label in state.arcs: if label in self.dfas: if label in self.first: fset = self.first[label] @@ -168,13 +167,12 @@ class ParserGenerator(object): if symbol in inverse: raise ValueError( "rule %s is ambiguous; %s is in the first sets of %s as well" - " as %s" - % (name, symbol, label, inverse[symbol]) + " as %s" % (name, symbol, label, inverse[symbol]) ) inverse[symbol] = label self.first[name] = totalset - def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]: + def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]: dfas = {} startsymbol: Optional[str] = None # MSTART: (NEWLINE | RULE)* ENDMARKER @@ -189,9 +187,9 @@ class ParserGenerator(object): # self.dump_nfa(name, a, z) dfa = self.make_dfa(a, z) # self.dump_dfa(name, dfa) - oldlen = len(dfa) + # oldlen = len(dfa) self.simplify_dfa(dfa) - newlen = len(dfa) + # newlen = len(dfa) dfas[name] = dfa # print name, oldlen, newlen if startsymbol is None: @@ -238,7 +236,7 @@ class ParserGenerator(object): state.addarc(st, label) return states # List of DFAState instances; first one is start - def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None: + def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: print("Dump of NFA for", name) todo = [start] for i, state in enumerate(todo): @@ -254,7 +252,7 @@ class ParserGenerator(object): else: print(" %s -> %d" % (label, j)) - def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None: + def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: print("Dump of DFA for", name) for i, state in enumerate(dfa): print(" State", i, state.isfinal and "(final)" or "") @@ -345,9 +343,9 @@ class ParserGenerator(object): self.raise_error( "expected (...) or NAME or STRING, got %s/%s", self.type, self.value ) - assert False + raise AssertionError - def expect(self, type: int, value: Optional[Any] = None) -> Text: + def expect(self, type: int, value: Optional[Any] = None) -> str: if self.type != type or (value is not None and self.value != value): self.raise_error( "expected %s/%s, got %s/%s", type, value, self.type, self.value @@ -367,27 +365,27 @@ class ParserGenerator(object): if args: try: msg = msg % args - except: + except Exception: msg = " ".join([msg] + list(map(str, args))) raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) -class NFAState(object): - arcs: List[Tuple[Optional[Text], "NFAState"]] +class NFAState: + arcs: List[Tuple[Optional[str], "NFAState"]] def __init__(self) -> None: self.arcs = [] # list of (label, NFAState) pairs - def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None: + def addarc(self, next: "NFAState", label: Optional[str] = None) -> None: assert label is None or isinstance(label, str) assert isinstance(next, NFAState) self.arcs.append((label, next)) -class DFAState(object): +class DFAState: nfaset: Dict[NFAState, Any] isfinal: bool - arcs: Dict[Text, "DFAState"] + arcs: Dict[str, "DFAState"] def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: assert isinstance(nfaset, dict) @@ -397,7 +395,7 @@ class DFAState(object): self.isfinal = final in nfaset self.arcs = {} # map from label to DFAState - def addarc(self, next: "DFAState", label: Text) -> None: + def addarc(self, next: "DFAState", label: str) -> None: assert isinstance(label, str) assert label not in self.arcs assert isinstance(next, DFAState)