import re
from codecs import BOM_UTF8, lookup
+from attr import dataclass
from blib2to3.pgen2.token import *
from . import token
tabsize = 8
+@dataclass(frozen=True)
+class TokenizerConfig:
+ async_is_reserved_keyword: bool = False
+
class TokenError(Exception): pass
class StopTokenizing(Exception): pass
ut = Untokenizer()
return ut.untokenize(iterable)
-def generate_tokens(readline):
+def generate_tokens(readline, config: TokenizerConfig = TokenizerConfig()):
"""
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
contline = None
indents = [0]
+ # If we know we're parsing 3.7+, we can unconditionally parse `async` and
+ # `await` as keywords.
+ async_is_reserved_keyword = config.async_is_reserved_keyword
# 'stashed' and 'async_*' are used for async/await parsing
stashed = None
async_def = False
yield (STRING, token, spos, epos, line)
elif initial.isidentifier(): # ordinary name
if token in ('async', 'await'):
- if async_def:
+ if async_is_reserved_keyword or async_def:
yield (ASYNC if token == 'async' else AWAIT,
token, spos, epos, line)
continue
stashed = tok
continue
- if token == 'def':
+ if token in ('def', 'for'):
if (stashed
and stashed[0] == NAME
and stashed[1] == 'async'):
- async_def = True
- async_def_indent = indents[-1]
+ if token == 'def':
+ async_def = True
+ async_def_indent = indents[-1]
yield (ASYNC, stashed[1],
stashed[2], stashed[3],