Decrease cost of ipynb code path when unneeded (#3748)

[etc/vim.git] / src / blib2to3 / pgen2 / pgen.py

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index a685145933c3c47cde23ef29996c29f81ec5bc36..b5ebc7b3e42c0b8aa1503214042c10bd7f964406 100644 (file)
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -8,7 +8,6 @@ from typing import (
     Any,
     Dict,
     IO,
     Any,
     Dict,
     IO,

-    Iterable,

     Iterator,
     List,
     Optional,
     Iterator,
     List,
     Optional,


@@ -31,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
 
 
 class ParserGenerator(object):

-

     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]


@@ -40,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:

-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")

             close_stream = stream.close
         self.filename = filename
         self.stream = stream
             close_stream = stream.close
         self.filename = filename
         self.stream = stream


@@ -116,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():

+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+

                 # A keyword
                 # A keyword

-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]

                 else:
                     c.labels.append((token.NAME, value))
                 else:
                     c.labels.append((token.NAME, value))

-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel

                     return ilabel
             else:
                 # An operator (any non-numeric token)
                     return ilabel
             else:
                 # An operator (any non-numeric token)