Declare support for Python 3.8 (#1069)

[etc/vim.git] / blib2to3 / pgen2 / grammar.py

diff --git a/blib2to3/pgen2/grammar.py b/blib2to3/pgen2/grammar.py
index 088c58bfa99c1b1b0ea5f269604e919e74685c3c..aa025cfd2543de34fe3eea6ec51be0a6138c9c9a 100644 (file)
--- a/blib2to3/pgen2/grammar.py
+++ b/blib2to3/pgen2/grammar.py
@@ -13,8 +13,9 @@ fallback token code OP, but the parser needs the actual token code.
 """
 
 # Python imports
 """
 
 # Python imports

-import collections
+import os

 import pickle
 import pickle

+import tempfile

 
 # Local imports
 from . import token
 
 # Local imports
 from . import token


@@ -84,23 +85,14 @@ class Grammar(object):
         self.tokens = {}
         self.symbol2label = {}
         self.start = 256
         self.tokens = {}
         self.symbol2label = {}
         self.start = 256

+        # Python 3.7+ parses async as a keyword, not an identifier
+        self.async_keywords = False

 
     def dump(self, filename):
 
     def dump(self, filename):

-        """Dump the grammar tables to a pickle file.
-
-        dump() recursively changes all dict to OrderedDict, so the pickled file
-        is not exactly the same as what was passed in to dump(). load() uses the
-        pickled file to create the tables, but  only changes OrderedDict to dict
-        at the top level; it does not recursively change OrderedDict to dict.
-        So, the loaded tables are different from the original tables that were
-        passed to load() in that some of the OrderedDict (from the pickled file)
-        are not changed back to dict. For parsing, this has no effect on
-        performance because OrderedDict uses dict's __getitem__ with nothing in
-        between.
-        """
-        with open(filename, "wb") as f:
-            d = _make_deterministic(self.__dict__)
-            pickle.dump(d, f, 2)
+        """Dump the grammar tables to a pickle file."""
+        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False) as f:
+            pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
+        os.replace(f.name, filename)

 
     def load(self, filename):
         """Load the grammar tables from a pickle file."""
 
     def load(self, filename):
         """Load the grammar tables from a pickle file."""


@@ -123,6 +115,7 @@ class Grammar(object):
         new.labels = self.labels[:]
         new.states = self.states[:]
         new.start = self.start
         new.labels = self.labels[:]
         new.states = self.states[:]
         new.start = self.start

+        new.async_keywords = self.async_keywords

         return new
 
     def report(self):
         return new
 
     def report(self):


@@ -141,17 +134,6 @@ class Grammar(object):
         print("start", self.start)
 
 
         print("start", self.start)
 
 

-def _make_deterministic(top):
-    if isinstance(top, dict):
-        return collections.OrderedDict(
-            sorted(((k, _make_deterministic(v)) for k, v in top.items())))
-    if isinstance(top, list):
-        return [_make_deterministic(e) for e in top]
-    if isinstance(top, tuple):
-        return tuple(_make_deterministic(e) for e in top)
-    return top
-
-

 # Map from operator to number (since tokenize doesn't do this)
 
 opmap_raw = """
 # Map from operator to number (since tokenize doesn't do this)
 
 opmap_raw = """


@@ -202,6 +184,7 @@ opmap_raw = """
 // DOUBLESLASH
 //= DOUBLESLASHEQUAL
 -> RARROW
 // DOUBLESLASH
 //= DOUBLESLASHEQUAL
 -> RARROW

+:= COLONEQUAL

 """
 
 opmap = {}
 """
 
 opmap = {}