Declare support for Python 3.8 (#1069)

[etc/vim.git] / blib2to3 / pgen2 / tokenize.py

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 43e1d597bc9b64792dd19229830f7578032fbe41..977548912c00f584ef135036d7185b67f289f47d 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -29,9 +29,8 @@ __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 
 __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 

-import re
+import regex as re

 from codecs import BOM_UTF8, lookup
 from codecs import BOM_UTF8, lookup

-from attr import dataclass

 from blib2to3.pgen2.token import *
 
 from . import token
 from blib2to3.pgen2.token import *
 
 from . import token


@@ -90,7 +89,7 @@ String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
                  r"//=?", r"->",
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
                  r"//=?", r"->",

-                 r"[+\-*/%&@|^=<>]=?",
+                 r"[+\-*/%&@|^=<>:]=?",

                  r"~")
 
 Bracket = '[][(){}]'
                  r"~")
 
 Bracket = '[][(){}]'


@@ -138,10 +137,6 @@ single_quoted = (
 
 tabsize = 8
 
 
 tabsize = 8
 

-@dataclass(frozen=True)
-class TokenizerConfig:
-    async_is_reserved_keyword: bool = False
-

 class TokenError(Exception): pass
 
 class StopTokenizing(Exception): pass
 class TokenError(Exception): pass
 
 class StopTokenizing(Exception): pass


@@ -339,7 +334,7 @@ def untokenize(iterable):
     ut = Untokenizer()
     return ut.untokenize(iterable)
 
     ut = Untokenizer()
     return ut.untokenize(iterable)
 

-def generate_tokens(readline, config: TokenizerConfig = TokenizerConfig()):
+def generate_tokens(readline, grammar=None):

     """
     The generate_tokens() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the
     """
     The generate_tokens() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the


@@ -363,7 +358,7 @@ def generate_tokens(readline, config: TokenizerConfig = TokenizerConfig()):
 
     # If we know we're parsing 3.7+, we can unconditionally parse `async` and
     # `await` as keywords.
 
     # If we know we're parsing 3.7+, we can unconditionally parse `async` and
     # `await` as keywords.

-    async_is_reserved_keyword = config.async_is_reserved_keyword
+    async_keywords = False if grammar is None else grammar.async_keywords

     # 'stashed' and 'async_*' are used for async/await parsing
     stashed = None
     async_def = False
     # 'stashed' and 'async_*' are used for async/await parsing
     stashed = None
     async_def = False


@@ -514,7 +509,7 @@ def generate_tokens(readline, config: TokenizerConfig = TokenizerConfig()):
                         yield (STRING, token, spos, epos, line)
                 elif initial.isidentifier():               # ordinary name
                     if token in ('async', 'await'):
                         yield (STRING, token, spos, epos, line)
                 elif initial.isidentifier():               # ordinary name
                     if token in ('async', 'await'):

-                        if async_is_reserved_keyword or async_def:
+                        if async_keywords or async_def:

                             yield (ASYNC if token == 'async' else AWAIT,
                                    token, spos, epos, line)
                             continue
                             yield (ASYNC if token == 'async' else AWAIT,
                                    token, spos, epos, line)
                             continue