Fix long lines with power operator(s) getting splitted before line length (#3942)

[etc/vim.git] / src / blib2to3 / pgen2 / tokenize.py

diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py
index 2d0cc4324ce75b7554500c96c1aea7ce69ea923b..d0607f4b1e17b202d4d19603287ed5cfc0a57288 100644 (file)
--- a/src/blib2to3/pgen2/tokenize.py
+++ b/src/blib2to3/pgen2/tokenize.py
@@ -30,32 +30,41 @@ each time a new token is found."""
 import sys
 from typing import (
     Callable,
 import sys
 from typing import (
     Callable,

+    Final,

     Iterable,
     Iterator,
     List,
     Optional,
     Iterable,
     Iterator,
     List,
     Optional,

+    Pattern,

     Set,
     Set,

-    Text,

     Tuple,
     Tuple,

-    Pattern,

     Union,
     cast,
 )
 
     Union,
     cast,
 )
 

-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
-
-from blib2to3.pgen2.token import *

 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.grammar import Grammar

+from blib2to3.pgen2.token import (
+    ASYNC,
+    AWAIT,
+    COMMENT,
+    DEDENT,
+    ENDMARKER,
+    ERRORTOKEN,
+    INDENT,
+    NAME,
+    NEWLINE,
+    NL,
+    NUMBER,
+    OP,
+    STRING,
+    tok_name,
+)

 
 __author__ = "Ka-Ping Yee <ping@lfw.org>"
 __credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
 
 import re
 from codecs import BOM_UTF8, lookup
 
 __author__ = "Ka-Ping Yee <ping@lfw.org>"
 __credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
 
 import re
 from codecs import BOM_UTF8, lookup

-from blib2to3.pgen2.token import *

 
 from . import token
 
 
 from . import token
 


@@ -80,7 +89,7 @@ def maybe(*choices: str) -> str:
 
 
 def _combinations(*l: str) -> Set[str]:
 
 
 def _combinations(*l: str) -> Set[str]:

-    return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold())
+    return {x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()}

 
 
 Whitespace = r"[ \f\t]*"
 
 
 Whitespace = r"[ \f\t]*"


@@ -192,7 +201,7 @@ Coord = Tuple[int, int]
 
 
 def printtoken(
 
 
 def printtoken(

-    type: int, token: Text, srow_col: Coord, erow_col: Coord, line: Text
+    type: int, token: str, srow_col: Coord, erow_col: Coord, line: str

 ) -> None:  # for testing
     (srow, scol) = srow_col
     (erow, ecol) = erow_col
 ) -> None:  # for testing
     (srow, scol) = srow_col
     (erow, ecol) = erow_col


@@ -201,10 +210,10 @@ def printtoken(
     )
 
 
     )
 
 

-TokenEater = Callable[[int, Text, Coord, Coord, Text], None]
+TokenEater = Callable[[int, str, Coord, Coord, str], None]

 
 
 
 

-def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) -> None:
+def tokenize(readline: Callable[[], str], tokeneater: TokenEater = printtoken) -> None:

     """
     The tokenize() function accepts two parameters: one representing the
     input stream, and one providing an output mechanism for tokenize().
     """
     The tokenize() function accepts two parameters: one representing the
     input stream, and one providing an output mechanism for tokenize().


@@ -224,17 +233,17 @@ def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken)
 
 
 # backwards compatible interface
 
 
 # backwards compatible interface

-def tokenize_loop(readline: Callable[[], Text], tokeneater: TokenEater) -> None:
+def tokenize_loop(readline: Callable[[], str], tokeneater: TokenEater) -> None:

     for token_info in generate_tokens(readline):
         tokeneater(*token_info)
 
 
     for token_info in generate_tokens(readline):
         tokeneater(*token_info)
 
 

-GoodTokenInfo = Tuple[int, Text, Coord, Coord, Text]
+GoodTokenInfo = Tuple[int, str, Coord, Coord, str]

 TokenInfo = Union[Tuple[int, str], GoodTokenInfo]
 
 
 class Untokenizer:
 TokenInfo = Union[Tuple[int, str], GoodTokenInfo]
 
 
 class Untokenizer:

-    tokens: List[Text]
+    tokens: List[str]

     prev_row: int
     prev_col: int
 
     prev_row: int
     prev_col: int
 


@@ -250,13 +259,13 @@ class Untokenizer:
         if col_offset:
             self.tokens.append(" " * col_offset)
 
         if col_offset:
             self.tokens.append(" " * col_offset)
 

-    def untokenize(self, iterable: Iterable[TokenInfo]) -> Text:
+    def untokenize(self, iterable: Iterable[TokenInfo]) -> str:

         for t in iterable:
             if len(t) == 2:
                 self.compat(cast(Tuple[int, str], t), iterable)
                 break
             tok_type, token, start, end, line = cast(
         for t in iterable:
             if len(t) == 2:
                 self.compat(cast(Tuple[int, str], t), iterable)
                 break
             tok_type, token, start, end, line = cast(

-                Tuple[int, Text, Coord, Coord, Text], t
+                Tuple[int, str, Coord, Coord, str], t

             )
             self.add_whitespace(start)
             self.tokens.append(token)
             )
             self.add_whitespace(start)
             self.tokens.append(token)


@@ -266,7 +275,7 @@ class Untokenizer:
                 self.prev_col = 0
         return "".join(self.tokens)
 
                 self.prev_col = 0
         return "".join(self.tokens)
 

-    def compat(self, token: Tuple[int, Text], iterable: Iterable[TokenInfo]) -> None:
+    def compat(self, token: Tuple[int, str], iterable: Iterable[TokenInfo]) -> None:

         startline = False
         indents = []
         toks_append = self.tokens.append
         startline = False
         indents = []
         toks_append = self.tokens.append


@@ -338,7 +347,7 @@ def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, List[bytes]]:
         try:
             return readline()
         except StopIteration:
         try:
             return readline()
         except StopIteration:

-            return bytes()
+            return b""

 
     def find_cookie(line: bytes) -> Optional[str]:
         try:
 
     def find_cookie(line: bytes) -> Optional[str]:
         try:


@@ -387,7 +396,7 @@ def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, List[bytes]]:
     return default, [first, second]
 
 
     return default, [first, second]
 
 

-def untokenize(iterable: Iterable[TokenInfo]) -> Text:
+def untokenize(iterable: Iterable[TokenInfo]) -> str:

     """Transform tokens back into Python source code.
 
     Each element returned by the iterable must be a token sequence
     """Transform tokens back into Python source code.
 
     Each element returned by the iterable must be a token sequence


@@ -410,7 +419,7 @@ def untokenize(iterable: Iterable[TokenInfo]) -> Text:
 
 
 def generate_tokens(
 
 
 def generate_tokens(

-    readline: Callable[[], Text], grammar: Optional[Grammar] = None
+    readline: Callable[[], str], grammar: Optional[Grammar] = None

 ) -> Iterator[GoodTokenInfo]:
     """
     The generate_tokens() generator requires one argument, readline, which
 ) -> Iterator[GoodTokenInfo]:
     """
     The generate_tokens() generator requires one argument, readline, which


@@ -680,14 +689,12 @@ def generate_tokens(
         yield stashed
         stashed = None
 
         yield stashed
         stashed = None
 

-    for indent in indents[1:]:  # pop remaining indent levels
+    for _indent in indents[1:]:  # pop remaining indent levels

         yield (DEDENT, "", (lnum, 0), (lnum, 0), "")
     yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "")
 
 
 if __name__ == "__main__":  # testing
         yield (DEDENT, "", (lnum, 0), (lnum, 0), "")
     yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "")
 
 
 if __name__ == "__main__":  # testing

-    import sys
-

     if len(sys.argv) > 1:
         tokenize(open(sys.argv[1]).readline)
     else:
     if len(sys.argv) > 1:
         tokenize(open(sys.argv[1]).readline)
     else: