]> git.madduck.net Git - etc/vim.git/blobdiff - blib2to3/pgen2/grammar.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Add option for printing a colored diff (#1266)
[etc/vim.git] / blib2to3 / pgen2 / grammar.py
index b20b9236b212d2886bbf986b90db1139afddae5e..2882cdac89b2ac138151edd8d1791ea55db799f0 100644 (file)
@@ -13,13 +13,20 @@ fallback token code OP, but the parser needs the actual token code.
 """
 
 # Python imports
-import os.path
+import os
 import pickle
 import tempfile
+from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar, Union
 
 # Local imports
 from . import token
 
+_P = TypeVar("_P", bound="Grammar")
+Label = Tuple[int, Optional[Text]]
+DFA = List[List[Tuple[int, int]]]
+DFAS = Tuple[DFA, Dict[int, int]]
+Path = Union[str, "os.PathLike[str]"]
+
 
 class Grammar(object):
     """Pgen parsing tables conversion class.
@@ -75,49 +82,74 @@ class Grammar(object):
 
     """
 
-    def __init__(self):
-        self.symbol2number = {}
-        self.number2symbol = {}
-        self.states = []
-        self.dfas = {}
-        self.labels = [(0, "EMPTY")]
-        self.keywords = {}
-        self.tokens = {}
-        self.symbol2label = {}
+    def __init__(self) -> None:
+        self.symbol2number: Dict[str, int] = {}
+        self.number2symbol: Dict[int, str] = {}
+        self.states: List[DFA] = []
+        self.dfas: Dict[int, DFAS] = {}
+        self.labels: List[Label] = [(0, "EMPTY")]
+        self.keywords: Dict[str, int] = {}
+        self.tokens: Dict[int, int] = {}
+        self.symbol2label: Dict[str, int] = {}
         self.start = 256
+        # Python 3.7+ parses async as a keyword, not an identifier
+        self.async_keywords = False
 
-    def dump(self, filename):
+    def dump(self, filename: Path) -> None:
         """Dump the grammar tables to a pickle file."""
-        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False) as f:
-            pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
+
+        # mypyc generates objects that don't have a __dict__, but they
+        # do have __getstate__ methods that will return an equivalent
+        # dictionary
+        if hasattr(self, "__dict__"):
+            d = self.__dict__
+        else:
+            d = self.__getstate__()  # type: ignore
+
+        with tempfile.NamedTemporaryFile(
+            dir=os.path.dirname(filename), delete=False
+        ) as f:
+            pickle.dump(d, f, pickle.HIGHEST_PROTOCOL)
         os.replace(f.name, filename)
 
-    def load(self, filename):
+    def _update(self, attrs: Dict[str, Any]) -> None:
+        for k, v in attrs.items():
+            setattr(self, k, v)
+
+    def load(self, filename: Path) -> None:
         """Load the grammar tables from a pickle file."""
         with open(filename, "rb") as f:
             d = pickle.load(f)
-        self.__dict__.update(d)
+        self._update(d)
 
-    def loads(self, pkl):
+    def loads(self, pkl: bytes) -> None:
         """Load the grammar tables from a pickle bytes object."""
-        self.__dict__.update(pickle.loads(pkl))
+        self._update(pickle.loads(pkl))
 
-    def copy(self):
+    def copy(self: _P) -> _P:
         """
         Copy the grammar.
         """
         new = self.__class__()
-        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
-                          "tokens", "symbol2label"):
+        for dict_attr in (
+            "symbol2number",
+            "number2symbol",
+            "dfas",
+            "keywords",
+            "tokens",
+            "symbol2label",
+        ):
             setattr(new, dict_attr, getattr(self, dict_attr).copy())
         new.labels = self.labels[:]
         new.states = self.states[:]
         new.start = self.start
+        new.async_keywords = self.async_keywords
         return new
 
-    def report(self):
+    def report(self) -> None:
         """Dump the grammar tables to standard output, for debugging."""
         from pprint import pprint
+
         print("s2n")
         pprint(self.symbol2number)
         print("n2s")
@@ -181,6 +213,7 @@ opmap_raw = """
 // DOUBLESLASH
 //= DOUBLESLASHEQUAL
 -> RARROW
+:= COLONEQUAL
 """
 
 opmap = {}