Document each configuration option in more detail (#2839)

[etc/vim.git] / src / blib2to3 / pgen2 / pgen.py

diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py
index 13ec51d1878f51bd73d9d09778dc5a5a4726ea6c..b5ebc7b3e42c0b8aa1503214042c10bd7f964406 100644 (file)
--- a/src/blib2to3/pgen2/pgen.py
+++ b/src/blib2to3/pgen2/pgen.py
@@ -8,7 +8,6 @@ from typing import (
     Any,
     Dict,
     IO,
-    Iterable,
     Iterator,
     List,
     Optional,
@@ -31,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
-
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
@@ -40,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -116,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)
@@ -168,8 +171,7 @@ class ParserGenerator(object):
                 if symbol in inverse:
                     raise ValueError(
                         "rule %s is ambiguous; %s is in the first sets of %s as well"
-                        " as %s"
-                        % (name, symbol, label, inverse[symbol])
+                        " as %s" % (name, symbol, label, inverse[symbol])
                     )
                 inverse[symbol] = label
         self.first[name] = totalset