]> git.madduck.net Git - etc/vim.git/blobdiff - src/blib2to3/pgen2/pgen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

exclude tests/data/.* from mypy (#3917)
[etc/vim.git] / src / blib2to3 / pgen2 / pgen.py
index 13ec51d1878f51bd73d9d09778dc5a5a4726ea6c..3ece9bb41edd51c7d2d82a317cb14dafcea4f1f4 100644 (file)
@@ -1,27 +1,22 @@
 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
 # Licensed to PSF under a Contributor Agreement.
 
 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
 # Licensed to PSF under a Contributor Agreement.
 
-# Pgen imports
-from . import grammar, token, tokenize
-
+import os
 from typing import (
 from typing import (
+    IO,
     Any,
     Dict,
     Any,
     Dict,
-    IO,
-    Iterable,
     Iterator,
     List,
     Iterator,
     List,
+    NoReturn,
     Optional,
     Optional,
-    Text,
+    Sequence,
     Tuple,
     Union,
     Tuple,
     Union,
-    Sequence,
-    NoReturn,
 )
 )
-from blib2to3.pgen2 import grammar
-from blib2to3.pgen2.tokenize import GoodTokenInfo
-import os
 
 
+from blib2to3.pgen2 import grammar, token, tokenize
+from blib2to3.pgen2.tokenize import GoodTokenInfo
 
 Path = Union[str, "os.PathLike[str]"]
 
 
 Path = Union[str, "os.PathLike[str]"]
 
@@ -30,17 +25,16 @@ class PgenGrammar(grammar.Grammar):
     pass
 
 
     pass
 
 
-class ParserGenerator(object):
-
+class ParserGenerator:
     filename: Path
     filename: Path
-    stream: IO[Text]
+    stream: IO[str]
     generator: Iterator[GoodTokenInfo]
     generator: Iterator[GoodTokenInfo]
-    first: Dict[Text, Optional[Dict[Text, int]]]
+    first: Dict[str, Optional[Dict[str, int]]]
 
 
-    def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
+    def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None:
         close_stream = None
         if stream is None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -77,7 +71,7 @@ class ParserGenerator(object):
         c.start = c.symbol2number[self.startsymbol]
         return c
 
         c.start = c.symbol2number[self.startsymbol]
         return c
 
-    def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]:
+    def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]:
         rawfirst = self.first[name]
         assert rawfirst is not None
         first = {}
         rawfirst = self.first[name]
         assert rawfirst is not None
         first = {}
@@ -87,7 +81,7 @@ class ParserGenerator(object):
             first[ilabel] = 1
         return first
 
             first[ilabel] = 1
         return first
 
-    def make_label(self, c: PgenGrammar, label: Text) -> int:
+    def make_label(self, c: PgenGrammar, label: str) -> int:
         # XXX Maybe this should be a method on a subclass of converter?
         ilabel = len(c.labels)
         if label[0].isalpha():
         # XXX Maybe this should be a method on a subclass of converter?
         ilabel = len(c.labels)
         if label[0].isalpha():
@@ -116,12 +110,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)
                     return ilabel
             else:
                 # An operator (any non-numeric token)
@@ -141,13 +140,13 @@ class ParserGenerator(object):
                 self.calcfirst(name)
             # print name, self.first[name].keys()
 
                 self.calcfirst(name)
             # print name, self.first[name].keys()
 
-    def calcfirst(self, name: Text) -> None:
+    def calcfirst(self, name: str) -> None:
         dfa = self.dfas[name]
         self.first[name] = None  # dummy to detect left recursion
         state = dfa[0]
         totalset: Dict[str, int] = {}
         overlapcheck = {}
         dfa = self.dfas[name]
         self.first[name] = None  # dummy to detect left recursion
         state = dfa[0]
         totalset: Dict[str, int] = {}
         overlapcheck = {}
-        for label, next in state.arcs.items():
+        for label in state.arcs:
             if label in self.dfas:
                 if label in self.first:
                     fset = self.first[label]
             if label in self.dfas:
                 if label in self.first:
                     fset = self.first[label]
@@ -168,13 +167,12 @@ class ParserGenerator(object):
                 if symbol in inverse:
                     raise ValueError(
                         "rule %s is ambiguous; %s is in the first sets of %s as well"
                 if symbol in inverse:
                     raise ValueError(
                         "rule %s is ambiguous; %s is in the first sets of %s as well"
-                        " as %s"
-                        % (name, symbol, label, inverse[symbol])
+                        " as %s" % (name, symbol, label, inverse[symbol])
                     )
                 inverse[symbol] = label
         self.first[name] = totalset
 
                     )
                 inverse[symbol] = label
         self.first[name] = totalset
 
-    def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]:
+    def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]:
         dfas = {}
         startsymbol: Optional[str] = None
         # MSTART: (NEWLINE | RULE)* ENDMARKER
         dfas = {}
         startsymbol: Optional[str] = None
         # MSTART: (NEWLINE | RULE)* ENDMARKER
@@ -189,9 +187,9 @@ class ParserGenerator(object):
             # self.dump_nfa(name, a, z)
             dfa = self.make_dfa(a, z)
             # self.dump_dfa(name, dfa)
             # self.dump_nfa(name, a, z)
             dfa = self.make_dfa(a, z)
             # self.dump_dfa(name, dfa)
-            oldlen = len(dfa)
+            oldlen = len(dfa)
             self.simplify_dfa(dfa)
             self.simplify_dfa(dfa)
-            newlen = len(dfa)
+            newlen = len(dfa)
             dfas[name] = dfa
             # print name, oldlen, newlen
             if startsymbol is None:
             dfas[name] = dfa
             # print name, oldlen, newlen
             if startsymbol is None:
@@ -238,7 +236,7 @@ class ParserGenerator(object):
                 state.addarc(st, label)
         return states  # List of DFAState instances; first one is start
 
                 state.addarc(st, label)
         return states  # List of DFAState instances; first one is start
 
-    def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None:
+    def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None:
         print("Dump of NFA for", name)
         todo = [start]
         for i, state in enumerate(todo):
         print("Dump of NFA for", name)
         todo = [start]
         for i, state in enumerate(todo):
@@ -254,7 +252,7 @@ class ParserGenerator(object):
                 else:
                     print("    %s -> %d" % (label, j))
 
                 else:
                     print("    %s -> %d" % (label, j))
 
-    def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None:
+    def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None:
         print("Dump of DFA for", name)
         for i, state in enumerate(dfa):
             print("  State", i, state.isfinal and "(final)" or "")
         print("Dump of DFA for", name)
         for i, state in enumerate(dfa):
             print("  State", i, state.isfinal and "(final)" or "")
@@ -345,9 +343,9 @@ class ParserGenerator(object):
             self.raise_error(
                 "expected (...) or NAME or STRING, got %s/%s", self.type, self.value
             )
             self.raise_error(
                 "expected (...) or NAME or STRING, got %s/%s", self.type, self.value
             )
-            assert False
+            raise AssertionError
 
 
-    def expect(self, type: int, value: Optional[Any] = None) -> Text:
+    def expect(self, type: int, value: Optional[Any] = None) -> str:
         if self.type != type or (value is not None and self.value != value):
             self.raise_error(
                 "expected %s/%s, got %s/%s", type, value, self.type, self.value
         if self.type != type or (value is not None and self.value != value):
             self.raise_error(
                 "expected %s/%s, got %s/%s", type, value, self.type, self.value
@@ -367,27 +365,27 @@ class ParserGenerator(object):
         if args:
             try:
                 msg = msg % args
         if args:
             try:
                 msg = msg % args
-            except:
+            except Exception:
                 msg = " ".join([msg] + list(map(str, args)))
         raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
 
 
                 msg = " ".join([msg] + list(map(str, args)))
         raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
 
 
-class NFAState(object):
-    arcs: List[Tuple[Optional[Text], "NFAState"]]
+class NFAState:
+    arcs: List[Tuple[Optional[str], "NFAState"]]
 
     def __init__(self) -> None:
         self.arcs = []  # list of (label, NFAState) pairs
 
 
     def __init__(self) -> None:
         self.arcs = []  # list of (label, NFAState) pairs
 
-    def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None:
+    def addarc(self, next: "NFAState", label: Optional[str] = None) -> None:
         assert label is None or isinstance(label, str)
         assert isinstance(next, NFAState)
         self.arcs.append((label, next))
 
 
         assert label is None or isinstance(label, str)
         assert isinstance(next, NFAState)
         self.arcs.append((label, next))
 
 
-class DFAState(object):
+class DFAState:
     nfaset: Dict[NFAState, Any]
     isfinal: bool
     nfaset: Dict[NFAState, Any]
     isfinal: bool
-    arcs: Dict[Text, "DFAState"]
+    arcs: Dict[str, "DFAState"]
 
     def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
         assert isinstance(nfaset, dict)
 
     def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
         assert isinstance(nfaset, dict)
@@ -397,7 +395,7 @@ class DFAState(object):
         self.isfinal = final in nfaset
         self.arcs = {}  # map from label to DFAState
 
         self.isfinal = final in nfaset
         self.arcs = {}  # map from label to DFAState
 
-    def addarc(self, next: "DFAState", label: Text) -> None:
+    def addarc(self, next: "DFAState", label: str) -> None:
         assert isinstance(label, str)
         assert label not in self.arcs
         assert isinstance(next, DFAState)
         assert isinstance(label, str)
         assert label not in self.arcs
         assert isinstance(next, DFAState)