X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/2082a325fdd14f0aabd88f7f12a20f9fb085c538..c99417ffe8aa51015f08a96220072fa0dbcce51d:/src/blib2to3/pgen2/pgen.py?ds=sidebyside

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index 13ec51d..b5ebc7b 100644
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -8,7 +8,6 @@ from typing import (
     Any,
     Dict,
     IO,
-    Iterable,
     Iterator,
     List,
     Optional,
@@ -31,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
-
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
@@ -40,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -116,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)
@@ -168,8 +171,7 @@ class ParserGenerator(object):
                 if symbol in inverse:
                     raise ValueError(
                         "rule %s is ambiguous; %s is in the first sets of %s as well"
-                        " as %s"
-                        % (name, symbol, label, inverse[symbol])
+                        " as %s" % (name, symbol, label, inverse[symbol])
                     )
                 inverse[symbol] = label
         self.first[name] = totalset