X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/c86fb362329a2b980ed94200e0652f9c74d56bbd..c7495b9aa098ef7a358fc74556359d21c6a4ba11:/blib2to3/pgen2/tokenize.py?ds=inline diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py index 9a7664b..a5c6462 100644 --- a/blib2to3/pgen2/tokenize.py +++ b/blib2to3/pgen2/tokenize.py @@ -89,7 +89,7 @@ String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", # recognized as two instances of =). Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", + r"[+\-*/%&@|^=<>:]=?", r"~") Bracket = '[][(){}]' @@ -334,7 +334,7 @@ def untokenize(iterable): ut = Untokenizer() return ut.untokenize(iterable) -def generate_tokens(readline): +def generate_tokens(readline, grammar=None): """ The generate_tokens() generator requires one argument, readline, which must be a callable object which provides the same interface as the @@ -356,6 +356,9 @@ def generate_tokens(readline): contline = None indents = [0] + # If we know we're parsing 3.7+, we can unconditionally parse `async` and + # `await` as keywords. + async_keywords = False if grammar is None else grammar.async_keywords # 'stashed' and 'async_*' are used for async/await parsing stashed = None async_def = False @@ -506,7 +509,7 @@ def generate_tokens(readline): yield (STRING, token, spos, epos, line) elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): - if async_def: + if async_keywords or async_def: yield (ASYNC if token == 'async' else AWAIT, token, spos, epos, line) continue @@ -516,13 +519,14 @@ def generate_tokens(readline): stashed = tok continue - if token == 'def': + if token in ('def', 'for'): if (stashed and stashed[0] == NAME and stashed[1] == 'async'): - async_def = True - async_def_indent = indents[-1] + if token == 'def': + async_def = True + async_def_indent = indents[-1] yield (ASYNC, stashed[1], stashed[2], stashed[3],