]> git.madduck.net Git - etc/vim.git/blobdiff - src/blib2to3/pgen2/pgen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Bump pypa/cibuildwheel from 2.12.3 to 2.13.0 (#3710)
[etc/vim.git] / src / blib2to3 / pgen2 / pgen.py
index 13ec51d1878f51bd73d9d09778dc5a5a4726ea6c..b5ebc7b3e42c0b8aa1503214042c10bd7f964406 100644 (file)
@@ -8,7 +8,6 @@ from typing import (
     Any,
     Dict,
     IO,
-    Iterable,
     Iterator,
     List,
     Optional,
@@ -31,7 +30,6 @@ class PgenGrammar(grammar.Grammar):
 
 
 class ParserGenerator(object):
-
     filename: Path
     stream: IO[Text]
     generator: Iterator[GoodTokenInfo]
@@ -40,7 +38,7 @@ class ParserGenerator(object):
     def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
         close_stream = None
         if stream is None:
-            stream = open(filename)
+            stream = open(filename, encoding="utf-8")
             close_stream = stream.close
         self.filename = filename
         self.stream = stream
@@ -116,12 +114,17 @@ class ParserGenerator(object):
             assert label[0] in ('"', "'"), label
             value = eval(label)
             if value[0].isalpha():
+                if label[0] == '"':
+                    keywords = c.soft_keywords
+                else:
+                    keywords = c.keywords
+
                 # A keyword
-                if value in c.keywords:
-                    return c.keywords[value]
+                if value in keywords:
+                    return keywords[value]
                 else:
                     c.labels.append((token.NAME, value))
-                    c.keywords[value] = ilabel
+                    keywords[value] = ilabel
                     return ilabel
             else:
                 # An operator (any non-numeric token)
@@ -168,8 +171,7 @@ class ParserGenerator(object):
                 if symbol in inverse:
                     raise ValueError(
                         "rule %s is ambiguous; %s is in the first sets of %s as well"
-                        " as %s"
-                        % (name, symbol, label, inverse[symbol])
+                        " as %s" % (name, symbol, label, inverse[symbol])
                     )
                 inverse[symbol] = label
         self.first[name] = totalset