Document pre-commit mirror (#3828)

[etc/vim.git] / src / blib2to3 / pgen2 / parse.py

diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py
index 8fe9667289796bf16d2186b17a81a3b8cf1a6ece..299cc24a15f4bd353d155b62ef645d8d6a9ef148 100644 (file)
--- a/src/blib2to3/pgen2/parse.py
+++ b/src/blib2to3/pgen2/parse.py
@@ -9,33 +9,32 @@ See Parser/parser.c in the Python distribution for additional info on
 how this parsing engine works.
 
 """
 how this parsing engine works.
 
 """

-import copy

 from contextlib import contextmanager
 from contextlib import contextmanager

-
-# Local imports
-from . import grammar, token, tokenize

 from typing import (
 from typing import (

-    cast,
+    TYPE_CHECKING,

     Any,
     Any,

-    Optional,
-    Text,
-    Union,
-    Tuple,
+    Callable,

     Dict,
     Dict,

-    List,

     Iterator,
     Iterator,

-    Callable,
+    List,
+    Optional,

     Set,
     Set,

-    TYPE_CHECKING,
+    Tuple,
+    Union,
+    cast,

 )
 )

+

 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.grammar import Grammar

-from blib2to3.pytree import convert, NL, Context, RawNode, Leaf, Node
+from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert
+
+# Local imports
+from . import grammar, token, tokenize

 
 if TYPE_CHECKING:
 
 if TYPE_CHECKING:

-    from blib2to3.driver import TokenProxy
+    from blib2to3.pgen2.driver import TokenProxy

 
 
 
 

-Results = Dict[Text, NL]
+Results = Dict[str, NL]

 Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]]
 DFA = List[List[Tuple[int, int]]]
 DFAS = Tuple[DFA, Dict[int, int]]
 Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]]
 DFA = List[List[Tuple[int, int]]]
 DFAS = Tuple[DFA, Dict[int, int]]


@@ -54,7 +53,7 @@ def stack_copy(
     stack: List[Tuple[DFAS, int, RawNode]]
 ) -> List[Tuple[DFAS, int, RawNode]]:
     """Nodeless stack copy."""
     stack: List[Tuple[DFAS, int, RawNode]]
 ) -> List[Tuple[DFAS, int, RawNode]]:
     """Nodeless stack copy."""

-    return [(copy.deepcopy(dfa), label, DUMMY_NODE) for dfa, label, _ in stack]
+    return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack]

 
 
 class Recorder:
 
 
 class Recorder:


@@ -100,7 +99,7 @@ class Recorder:
         finally:
             self.parser.is_backtracking = is_backtracking
 
         finally:
             self.parser.is_backtracking = is_backtracking
 

-    def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None:
+    def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None:

         func: Callable[..., Any]
         if raw:
             func = self.parser._addtoken
         func: Callable[..., Any]
         if raw:
             func = self.parser._addtoken


@@ -114,7 +113,9 @@ class Recorder:
                     args.insert(0, ilabel)
                 func(*args)
 
                     args.insert(0, ilabel)
                 func(*args)
 

-    def determine_route(self, value: Text = None, force: bool = False) -> Optional[int]:
+    def determine_route(
+        self, value: Optional[str] = None, force: bool = False
+    ) -> Optional[int]:

         alive_ilabels = self.ilabels
         if len(alive_ilabels) == 0:
             *_, most_successful_ilabel = self._dead_ilabels
         alive_ilabels = self.ilabels
         if len(alive_ilabels) == 0:
             *_, most_successful_ilabel = self._dead_ilabels


@@ -131,10 +132,10 @@ class ParseError(Exception):
     """Exception to signal the parser is stuck."""
 
     def __init__(
     """Exception to signal the parser is stuck."""
 
     def __init__(

-        self, msg: Text, type: Optional[int], value: Optional[Text], context: Context
+        self, msg: str, type: Optional[int], value: Optional[str], context: Context

     ) -> None:
         Exception.__init__(
     ) -> None:
         Exception.__init__(

-            self, "%s: type=%r, value=%r, context=%r" % (msg, type, value, context)
+            self, f"{msg}: type={type!r}, value={value!r}, context={context!r}"

         )
         self.msg = msg
         self.type = type
         )
         self.msg = msg
         self.type = type


@@ -142,7 +143,7 @@ class ParseError(Exception):
         self.context = context
 
 
         self.context = context
 
 

-class Parser(object):
+class Parser:

     """Parser engine.
 
     The proper usage sequence is:
     """Parser engine.
 
     The proper usage sequence is:


@@ -236,7 +237,7 @@ class Parser(object):
         self.used_names: Set[str] = set()
         self.proxy = proxy
 
         self.used_names: Set[str] = set()
         self.proxy = proxy
 

-    def addtoken(self, type: int, value: Text, context: Context) -> bool:
+    def addtoken(self, type: int, value: str, context: Context) -> bool:

         """Add a token; return True iff this is the end of the program."""
         # Map from token to label
         ilabels = self.classify(type, value, context)
         """Add a token; return True iff this is the end of the program."""
         # Map from token to label
         ilabels = self.classify(type, value, context)


@@ -269,6 +270,10 @@ class Parser(object):
                     break
 
                 next_token_type, next_token_value, *_ = proxy.eat(counter)
                     break
 
                 next_token_type, next_token_value, *_ = proxy.eat(counter)

+                if next_token_type in (tokenize.COMMENT, tokenize.NL):
+                    counter += 1
+                    continue
+

                 if next_token_type == tokenize.OP:
                     next_token_type = grammar.opmap[next_token_value]
 
                 if next_token_type == tokenize.OP:
                     next_token_type = grammar.opmap[next_token_value]
 


@@ -280,7 +285,7 @@ class Parser(object):
 
         return self._addtoken(ilabel, type, value, context)
 
 
         return self._addtoken(ilabel, type, value, context)
 

-    def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bool:
+    def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> bool:

         # Loop until the token is shifted; may raise exceptions
         while True:
             dfa, state, node = self.stack[-1]
         # Loop until the token is shifted; may raise exceptions
         while True:
             dfa, state, node = self.stack[-1]


@@ -325,7 +330,7 @@ class Parser(object):
                     # No success finding a transition
                     raise ParseError("bad input", type, value, context)
 
                     # No success finding a transition
                     raise ParseError("bad input", type, value, context)
 

-    def classify(self, type: int, value: Text, context: Context) -> List[int]:
+    def classify(self, type: int, value: str, context: Context) -> List[int]:

         """Turn a token into a label.  (Internal)
 
         Depending on whether the value is a soft-keyword or not,
         """Turn a token into a label.  (Internal)
 
         Depending on whether the value is a soft-keyword or not,


@@ -348,7 +353,7 @@ class Parser(object):
             raise ParseError("bad token", type, value, context)
         return [ilabel]
 
             raise ParseError("bad token", type, value, context)
         return [ilabel]
 

-    def shift(self, type: int, value: Text, newstate: int, context: Context) -> None:
+    def shift(self, type: int, value: str, newstate: int, context: Context) -> None:

         """Shift a token.  (Internal)"""
         if self.is_backtracking:
             dfa, state, _ = self.stack[-1]
         """Shift a token.  (Internal)"""
         if self.is_backtracking:
             dfa, state, _ = self.stack[-1]