X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/b3ceb293d9e69295a190fed93517cbe1b7372154..eedfc3832290b3a32825b3c0f2dfa3f3d7ee9d1c:/src/blib2to3/pgen2/pgen.py?ds=sidebyside diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 564ebbd..b5ebc7b 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -30,7 +30,6 @@ class PgenGrammar(grammar.Grammar): class ParserGenerator(object): - filename: Path stream: IO[Text] generator: Iterator[GoodTokenInfo] @@ -39,7 +38,7 @@ class ParserGenerator(object): def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: close_stream = None if stream is None: - stream = open(filename) + stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename self.stream = stream @@ -115,12 +114,17 @@ class ParserGenerator(object): assert label[0] in ('"', "'"), label value = eval(label) if value[0].isalpha(): + if label[0] == '"': + keywords = c.soft_keywords + else: + keywords = c.keywords + # A keyword - if value in c.keywords: - return c.keywords[value] + if value in keywords: + return keywords[value] else: c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel + keywords[value] = ilabel return ilabel else: # An operator (any non-numeric token)