X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/b3ceb293d9e69295a190fed93517cbe1b7372154..7be273531852a16a407fcb66d5efeede0f7ca474:/src/blib2to3/pgen2/pgen.py?ds=inline

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index 564ebbd..b5ebc7b 100644
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -30,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
-
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
@@ -39,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -115,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)