X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/e3ccabb23c5dc5495bd8f96b5c90c1db6a350d6d..e7c3368c1316c38338cef34fffc42ea3252b1802:/src/blib2to3/pgen2/pgen.py

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index a685145..3ece9bb 100644
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -1,27 +1,22 @@
 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
 # Licensed to PSF under a Contributor Agreement.
 
-# Pgen imports
-from . import grammar, token, tokenize
-
+import os
 from typing import (
+    IO,
     Any,
     Dict,
-    IO,
-    Iterable,
     Iterator,
     List,
+    NoReturn,
     Optional,
-    Text,
+    Sequence,
     Tuple,
     Union,
-    Sequence,
-    NoReturn,
 )
-from blib2to3.pgen2 import grammar
-from blib2to3.pgen2.tokenize import GoodTokenInfo
-import os
 
+from blib2to3.pgen2 import grammar, token, tokenize
+from blib2to3.pgen2.tokenize import GoodTokenInfo
 
 Path = Union[str, "os.PathLike[str]"]
 
@@ -30,17 +25,16 @@ class PgenGrammar(grammar.Grammar):
     pass
 
 
-class ParserGenerator(object):
-
+class ParserGenerator:
     filename: Path
-    stream: IO[Text]
+    stream: IO[str]
     generator: Iterator[GoodTokenInfo]
-    first: Dict[Text, Optional[Dict[Text, int]]]
+    first: Dict[str, Optional[Dict[str, int]]]
 
-    def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
+    def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -77,7 +71,7 @@ class ParserGenerator(object):
         c.start = c.symbol2number[self.startsymbol]
         return c
 
-    def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]:
+    def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]:
         rawfirst = self.first[name]
         assert rawfirst is not None
         first = {}
@@ -87,7 +81,7 @@ class ParserGenerator(object):
             first[ilabel] = 1
         return first
 
-    def make_label(self, c: PgenGrammar, label: Text) -> int:
+    def make_label(self, c: PgenGrammar, label: str) -> int:
         # XXX Maybe this should be a method on a subclass of converter?
         ilabel = len(c.labels)
         if label[0].isalpha():
@@ -116,12 +110,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)
@@ -141,13 +140,13 @@ class ParserGenerator(object):
                 self.calcfirst(name)
             # print name, self.first[name].keys()
 
-    def calcfirst(self, name: Text) -> None:
+    def calcfirst(self, name: str) -> None:
         dfa = self.dfas[name]
         self.first[name] = None  # dummy to detect left recursion
         state = dfa[0]
         totalset: Dict[str, int] = {}
         overlapcheck = {}
-        for label, next in state.arcs.items():
+        for label in state.arcs:
             if label in self.dfas:
                 if label in self.first:
                     fset = self.first[label]
@@ -173,7 +172,7 @@ class ParserGenerator(object):
                 inverse[symbol] = label
         self.first[name] = totalset
 
-    def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]:
+    def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]:
         dfas = {}
         startsymbol: Optional[str] = None
         # MSTART: (NEWLINE | RULE)* ENDMARKER
@@ -188,9 +187,9 @@ class ParserGenerator(object):
             # self.dump_nfa(name, a, z)
             dfa = self.make_dfa(a, z)
             # self.dump_dfa(name, dfa)
-            oldlen = len(dfa)
+            # oldlen = len(dfa)
             self.simplify_dfa(dfa)
-            newlen = len(dfa)
+            # newlen = len(dfa)
             dfas[name] = dfa
             # print name, oldlen, newlen
             if startsymbol is None:
@@ -237,7 +236,7 @@ class ParserGenerator(object):
                 state.addarc(st, label)
         return states  # List of DFAState instances; first one is start
 
-    def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None:
+    def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None:
         print("Dump of NFA for", name)
         todo = [start]
         for i, state in enumerate(todo):
@@ -253,7 +252,7 @@ class ParserGenerator(object):
                 else:
                     print("    %s -> %d" % (label, j))
 
-    def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None:
+    def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None:
         print("Dump of DFA for", name)
         for i, state in enumerate(dfa):
             print("  State", i, state.isfinal and "(final)" or "")
@@ -344,9 +343,9 @@ class ParserGenerator(object):
             self.raise_error(
                 "expected (...) or NAME or STRING, got %s/%s", self.type, self.value
             )
-            assert False
+            raise AssertionError
 
-    def expect(self, type: int, value: Optional[Any] = None) -> Text:
+    def expect(self, type: int, value: Optional[Any] = None) -> str:
         if self.type != type or (value is not None and self.value != value):
             self.raise_error(
                 "expected %s/%s, got %s/%s", type, value, self.type, self.value
@@ -366,27 +365,27 @@ class ParserGenerator(object):
         if args:
             try:
                 msg = msg % args
-            except:
+            except Exception:
                 msg = " ".join([msg] + list(map(str, args)))
         raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
 
 
-class NFAState(object):
-    arcs: List[Tuple[Optional[Text], "NFAState"]]
+class NFAState:
+    arcs: List[Tuple[Optional[str], "NFAState"]]
 
     def __init__(self) -> None:
         self.arcs = []  # list of (label, NFAState) pairs
 
-    def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None:
+    def addarc(self, next: "NFAState", label: Optional[str] = None) -> None:
         assert label is None or isinstance(label, str)
         assert isinstance(next, NFAState)
         self.arcs.append((label, next))
 
 
-class DFAState(object):
+class DFAState:
     nfaset: Dict[NFAState, Any]
     isfinal: bool
-    arcs: Dict[Text, "DFAState"]
+    arcs: Dict[str, "DFAState"]
 
     def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
         assert isinstance(nfaset, dict)
@@ -396,7 +395,7 @@ class DFAState(object):
         self.isfinal = final in nfaset
         self.arcs = {}  # map from label to DFAState
 
-    def addarc(self, next: "DFAState", label: Text) -> None:
+    def addarc(self, next: "DFAState", label: str) -> None:
         assert isinstance(label, str)
         assert label not in self.arcs
         assert isinstance(next, DFAState)