X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/117891878e5be4d6b771ae5de299e51b679cea27..751583a1dfc691423d96d7711a4c8d9cfe3ee9c8:/src/blib2to3/pytree.py?ds=sidebyside diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index bd86270..2a0cd6d 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -10,21 +10,21 @@ even the comments and whitespace between tokens. There's also a pattern matching implementation here. """ -# mypy: allow-untyped-defs +# mypy: allow-untyped-defs, allow-incomplete-defs from typing import ( Any, Dict, + Iterable, Iterator, List, Optional, - Text, + Set, Tuple, TypeVar, Union, - Set, - Iterable, ) + from blib2to3.pgen2.grammar import Grammar __author__ = "Guido van Rossum " @@ -34,10 +34,10 @@ from io import StringIO HUGE: int = 0x7FFFFFFF # maximum repeat count, default max -_type_reprs: Dict[int, Union[Text, int]] = {} +_type_reprs: Dict[int, Union[str, int]] = {} -def type_repr(type_num: int) -> Union[Text, int]: +def type_repr(type_num: int) -> Union[str, int]: global _type_reprs if not _type_reprs: from .pygram import python_symbols @@ -54,12 +54,11 @@ def type_repr(type_num: int) -> Union[Text, int]: _P = TypeVar("_P", bound="Base") NL = Union["Node", "Leaf"] -Context = Tuple[Text, Tuple[int, int]] -RawNode = Tuple[int, Optional[Text], Optional[Context], Optional[List[NL]]] - +Context = Tuple[str, Tuple[int, int]] +RawNode = Tuple[int, Optional[str], Optional[Context], Optional[List[NL]]] -class Base(object): +class Base: """ Abstract base class for Node and Leaf. @@ -92,7 +91,7 @@ class Base(object): return self._eq(other) @property - def prefix(self) -> Text: + def prefix(self) -> str: raise NotImplementedError def _eq(self: _P, other: _P) -> bool: @@ -225,7 +224,7 @@ class Base(object): return 0 return 1 + self.parent.depth() - def get_suffix(self) -> Text: + def get_suffix(self) -> str: """ Return the string immediately following the invocant node. This is effectively equivalent to node.next_sibling.prefix @@ -238,18 +237,17 @@ class Base(object): class Node(Base): - """Concrete implementation for interior nodes.""" fixers_applied: Optional[List[Any]] - used_names: Optional[Set[Text]] + used_names: Optional[Set[str]] def __init__( self, type: int, children: List[NL], context: Optional[Any] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: Optional[List[Any]] = None, ) -> None: """ @@ -274,16 +272,16 @@ class Node(Base): else: self.fixers_applied = None - def __repr__(self) -> Text: + def __repr__(self) -> str: """Return a canonical string representation.""" assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, type_repr(self.type), self.children, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -291,7 +289,7 @@ class Node(Base): """ return "".join(map(str, self.children)) - def _eq(self, other) -> bool: + def _eq(self, other: Base) -> bool: """Compare two nodes for equality.""" return (self.type, self.children) == (other.type, other.children) @@ -317,7 +315,7 @@ class Node(Base): yield from child.pre_order() @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this node in the input. """ @@ -326,7 +324,7 @@ class Node(Base): return self.children[0].prefix @prefix.setter - def prefix(self, prefix) -> None: + def prefix(self, prefix: str) -> None: if self.children: self.children[0].prefix = prefix @@ -379,26 +377,32 @@ class Node(Base): class Leaf(Base): - """Concrete implementation for leaf nodes.""" # Default values for instance variables - value: Text + value: str fixers_applied: List[Any] bracket_depth: int - opening_bracket: "Leaf" - used_names: Optional[Set[Text]] + # Changed later in brackets.py + opening_bracket: Optional["Leaf"] = None + used_names: Optional[Set[str]] _prefix = "" # Whitespace and comments preceding this token in the input lineno: int = 0 # Line where this token starts in the input column: int = 0 # Column where this token starts in the input + # If not None, this Leaf is created by converting a block of fmt off/skip + # code, and `fmt_pass_converted_first_leaf` points to the first Leaf in the + # converted code. + fmt_pass_converted_first_leaf: Optional["Leaf"] = None def __init__( self, type: int, - value: Text, + value: str, context: Optional[Context] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: List[Any] = [], + opening_bracket: Optional["Leaf"] = None, + fmt_pass_converted_first_leaf: Optional["Leaf"] = None, ) -> None: """ Initializer. @@ -416,19 +420,21 @@ class Leaf(Base): self._prefix = prefix self.fixers_applied: Optional[List[Any]] = fixers_applied[:] self.children = [] + self.opening_bracket = opening_bracket + self.fmt_pass_converted_first_leaf = fmt_pass_converted_first_leaf def __repr__(self) -> str: """Return a canonical string representation.""" from .pgen2.token import tok_name assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, tok_name.get(self.type, self.type), self.value, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -436,7 +442,7 @@ class Leaf(Base): """ return self._prefix + str(self.value) - def _eq(self, other) -> bool: + def _eq(self, other: "Leaf") -> bool: """Compare two nodes for equality.""" return (self.type, self.value) == (other.type, other.value) @@ -462,14 +468,14 @@ class Leaf(Base): yield self @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this token in the input. """ return self._prefix @prefix.setter - def prefix(self, prefix) -> None: + def prefix(self, prefix: str) -> None: self.changed() self._prefix = prefix @@ -494,11 +500,10 @@ def convert(gr: Grammar, raw_node: RawNode) -> NL: return Leaf(type, value or "", context=context) -_Results = Dict[Text, NL] - +_Results = Dict[str, NL] -class BasePattern(object): +class BasePattern: """ A pattern is a tree matching pattern. @@ -517,19 +522,19 @@ class BasePattern(object): type: Optional[int] type = None # Node type (token if < 256, symbol if >= 256) content: Any = None # Optional content matching pattern - name: Optional[Text] = None # Optional name used to store match in results dict + name: Optional[str] = None # Optional name used to store match in results dict def __new__(cls, *args, **kwds): """Constructor that prevents BasePattern from being instantiated.""" assert cls is not BasePattern, "Cannot instantiate BasePattern" return object.__new__(cls) - def __repr__(self) -> Text: + def __repr__(self) -> str: assert self.type is not None args = [type_repr(self.type), self.content, self.name] while args and args[-1] is None: del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) + return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args))) def _submatch(self, node, results=None) -> bool: raise NotImplementedError @@ -593,8 +598,8 @@ class LeafPattern(BasePattern): def __init__( self, type: Optional[int] = None, - content: Optional[Text] = None, - name: Optional[Text] = None, + content: Optional[str] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -615,7 +620,7 @@ class LeafPattern(BasePattern): self.content = content self.name = name - def match(self, node: NL, results=None): + def match(self, node: NL, results=None) -> bool: """Override match() to insist on a leaf node.""" if not isinstance(node, Leaf): return False @@ -638,14 +643,13 @@ class LeafPattern(BasePattern): class NodePattern(BasePattern): - wildcards: bool = False def __init__( self, type: Optional[int] = None, - content: Optional[Iterable[Text]] = None, - name: Optional[Text] = None, + content: Optional[Iterable[str]] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -675,7 +679,7 @@ class NodePattern(BasePattern): if isinstance(item, WildcardPattern): # type: ignore[unreachable] self.wildcards = True # type: ignore[unreachable] self.type = type - self.content = newcontent + self.content = newcontent # TODO: this is unbound when content is None self.name = name def _submatch(self, node, results=None) -> bool: @@ -707,7 +711,6 @@ class NodePattern(BasePattern): class WildcardPattern(BasePattern): - """ A wildcard pattern can match zero or more nodes. @@ -725,10 +728,10 @@ class WildcardPattern(BasePattern): def __init__( self, - content: Optional[Text] = None, + content: Optional[str] = None, min: int = 0, max: int = HUGE, - name: Optional[Text] = None, + name: Optional[str] = None, ) -> None: """ Initializer. @@ -917,7 +920,7 @@ class WildcardPattern(BasePattern): class NegatedPattern(BasePattern): - def __init__(self, content: Optional[Any] = None) -> None: + def __init__(self, content: Optional[BasePattern] = None) -> None: """ Initializer. @@ -938,7 +941,7 @@ class NegatedPattern(BasePattern): # We only match an empty sequence of nodes in its entirety return len(nodes) == 0 - def generate_matches(self, nodes) -> Iterator[Tuple[int, _Results]]: + def generate_matches(self, nodes: List[NL]) -> Iterator[Tuple[int, _Results]]: if self.content is None: # Return a match if there is an empty sequence if len(nodes) == 0: