Used by: add pandas and Pillow (#1057)

[etc/vim.git] / blib2to3 / pgen2 / tokenize.py

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index a352713f7deaba741e1d13c2a10da43be544d26a..a5c6462e7cfaeeaf47f5b685c723d1f079d5bbc2 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -29,7 +29,7 @@ __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 
 __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 

-import string, re, unicodedata
+import re

 from codecs import BOM_UTF8, lookup
 from blib2to3.pgen2.token import *
 
 from codecs import BOM_UTF8, lookup
 from blib2to3.pgen2.token import *
 


@@ -56,7 +56,7 @@ def _combinations(*l):
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)

-Name = r'[^\d\W]\w*'
+Name = r'\w+'  # this is invalid but it's fine because Name comes after Number in all groups

 
 Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
 Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
 
 Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
 Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'


@@ -89,7 +89,7 @@ String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
                  r"//=?", r"->",
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
                  r"//=?", r"->",

-                 r"[+\-*/%&@|^=<>]=?",
+                 r"[+\-*/%&@|^=<>:]=?",

                  r"~")
 
 Bracket = '[][(){}]'
                  r"~")
 
 Bracket = '[][(){}]'


@@ -334,9 +334,7 @@ def untokenize(iterable):
     ut = Untokenizer()
     return ut.untokenize(iterable)
 
     ut = Untokenizer()
     return ut.untokenize(iterable)
 

-InitialCategories = {'Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl', 'Mn', 'Mc', 'Nd', 'Pc'}
-
-def generate_tokens(readline):
+def generate_tokens(readline, grammar=None):

     """
     The generate_tokens() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the
     """
     The generate_tokens() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the


@@ -353,11 +351,14 @@ def generate_tokens(readline):
     logical line; continuation lines are included.
     """
     lnum = parenlev = continued = 0
     logical line; continuation lines are included.
     """
     lnum = parenlev = continued = 0

-    namechars, numchars = string.ascii_letters + '_', '0123456789'
+    numchars = '0123456789'

     contstr, needcont = '', 0
     contline = None
     indents = [0]
 
     contstr, needcont = '', 0
     contline = None
     indents = [0]
 

+    # If we know we're parsing 3.7+, we can unconditionally parse `async` and
+    # `await` as keywords.
+    async_keywords = False if grammar is None else grammar.async_keywords

     # 'stashed' and 'async_*' are used for async/await parsing
     stashed = None
     async_def = False
     # 'stashed' and 'async_*' are used for async/await parsing
     stashed = None
     async_def = False


@@ -506,10 +507,9 @@ def generate_tokens(readline):
                             yield stashed
                             stashed = None
                         yield (STRING, token, spos, epos, line)
                             yield stashed
                             stashed = None
                         yield (STRING, token, spos, epos, line)

-                elif (initial in namechars or              # ordinary name
-                      unicodedata.category(initial) in InitialCategories):
+                elif initial.isidentifier():               # ordinary name

                     if token in ('async', 'await'):
                     if token in ('async', 'await'):

-                        if async_def:
+                        if async_keywords or async_def:

                             yield (ASYNC if token == 'async' else AWAIT,
                                    token, spos, epos, line)
                             continue
                             yield (ASYNC if token == 'async' else AWAIT,
                                    token, spos, epos, line)
                             continue


@@ -519,13 +519,14 @@ def generate_tokens(readline):
                         stashed = tok
                         continue
 
                         stashed = tok
                         continue
 

-                    if token == 'def':
+                    if token in ('def', 'for'):

                         if (stashed
                                 and stashed[0] == NAME
                                 and stashed[1] == 'async'):
 
                         if (stashed
                                 and stashed[0] == NAME
                                 and stashed[1] == 'async'):
 

-                            async_def = True
-                            async_def_indent = indents[-1]
+                            if token == 'def':
+                                async_def = True
+                                async_def_indent = indents[-1]

 
                             yield (ASYNC, stashed[1],
                                    stashed[2], stashed[3],
 
                             yield (ASYNC, stashed[1],
                                    stashed[2], stashed[3],