X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/e3ccabb23c5dc5495bd8f96b5c90c1db6a350d6d..31b3b6701d2cfae072900f9d45dc8f1737ab282b:/src/blib2to3/pgen2/pgen.py?ds=sidebyside

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index a685145..b5ebc7b 100644
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -8,7 +8,6 @@ from typing import (
     Any,
     Dict,
     IO,
-    Iterable,
     Iterator,
     List,
     Optional,
@@ -31,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
-
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
@@ -40,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -116,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)