All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 blib2to3 Node/Leaf transformation-related utility functions.
18 if sys.version_info < (3, 8):
19 from typing_extensions import Final
21 from typing import Final
24 from blib2to3.pytree import Node, Leaf, type_repr
25 from blib2to3 import pygram
26 from blib2to3.pgen2 import token
28 from black.cache import CACHE_DIR
29 from black.strings import has_triple_quotes
32 pygram.initialize(CACHE_DIR)
33 syms = pygram.python_symbols
38 LN = Union[Leaf, Node]
43 WHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE}
54 STANDALONE_COMMENT: Final = 153
55 token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT"
56 LOGIC_OPERATORS: Final = {"and", "or"}
57 COMPARATORS: Final = {
65 MATH_OPERATORS: Final = {
81 STARS: Final = {token.STAR, token.DOUBLESTAR}
82 VARARGS_SPECIALS: Final = STARS | {token.SLASH}
83 VARARGS_PARENTS: Final = {
85 syms.argument, # double star in arglist
86 syms.trailer, # single argument to call
88 syms.varargslist, # lambdas
90 UNPACKING_PARENTS: Final = {
91 syms.atom, # single element of a list or set literal
95 syms.testlist_star_expr,
97 TEST_DESCENDANTS: Final = {
114 ASSIGNMENTS: Final = {
131 IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist}
132 BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE}
133 OPENING_BRACKETS = set(BRACKET.keys())
134 CLOSING_BRACKETS = set(BRACKET.values())
135 BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS
136 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
139 class Visitor(Generic[T]):
140 """Basic lib2to3 visitor that yields things of type `T` on `visit()`."""
142 def visit(self, node: LN) -> Iterator[T]:
143 """Main method to visit `node` and its children.
145 It tries to find a `visit_*()` method for the given `node.type`, like
146 `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.
147 If no dedicated `visit_*()` method is found, chooses `visit_default()`
150 Then yields objects of type `T` from the selected visitor.
153 name = token.tok_name[node.type]
155 name = str(type_repr(node.type))
156 # We explicitly branch on whether a visitor exists (instead of
157 # using self.visit_default as the default arg to getattr) in order
158 # to save needing to create a bound method object and so mypyc can
159 # generate a native call to visit_default.
160 visitf = getattr(self, f"visit_{name}", None)
162 yield from visitf(node)
164 yield from self.visit_default(node)
166 def visit_default(self, node: LN) -> Iterator[T]:
167 """Default `visit_*()` implementation. Recurses to children of `node`."""
168 if isinstance(node, Node):
169 for child in node.children:
170 yield from self.visit(child)
173 def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901
174 """Return whitespace prefix if needed for the given `leaf`.
176 `complex_subscript` signals whether the given leaf is part of a subscription
177 which has non-trivial arguments, like arithmetic expressions or function calls.
185 if t in ALWAYS_NO_SPACE:
188 if t == token.COMMENT:
191 assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
192 if t == token.COLON and p.type not in {
199 prev = leaf.prev_sibling
201 prevp = preceding_leaf(p)
202 if not prevp or prevp.type in OPENING_BRACKETS:
206 if prevp.type == token.COLON:
209 elif prevp.type != token.COMMA and not complex_subscript:
214 if prevp.type == token.EQUAL:
216 if prevp.parent.type in {
224 elif prevp.parent.type == syms.typedargslist:
225 # A bit hacky: if the equal sign has whitespace, it means we
226 # previously found it's a typed argument. So, we're using
230 elif prevp.type in VARARGS_SPECIALS:
231 if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
234 elif prevp.type == token.COLON:
235 if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
236 return SPACE if complex_subscript else NO
240 and prevp.parent.type == syms.factor
241 and prevp.type in MATH_OPERATORS
246 prevp.type == token.RIGHTSHIFT
248 and prevp.parent.type == syms.shift_expr
249 and prevp.prev_sibling
250 and prevp.prev_sibling.type == token.NAME
251 and prevp.prev_sibling.value == "print" # type: ignore
253 # Python 2 print chevron
255 elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator:
256 # no space in decorators
259 elif prev.type in OPENING_BRACKETS:
262 if p.type in {syms.parameters, syms.arglist}:
263 # untyped function signatures or calls
264 if not prev or prev.type != token.COMMA:
267 elif p.type == syms.varargslist:
269 if prev and prev.type != token.COMMA:
272 elif p.type == syms.typedargslist:
273 # typed function signatures
278 if prev.type != syms.tname:
281 elif prev.type == token.EQUAL:
282 # A bit hacky: if the equal sign has whitespace, it means we
283 # previously found it's a typed argument. So, we're using that, too.
286 elif prev.type != token.COMMA:
289 elif p.type == syms.tname:
292 prevp = preceding_leaf(p)
293 if not prevp or prevp.type != token.COMMA:
296 elif p.type == syms.trailer:
297 # attributes and calls
298 if t == token.LPAR or t == token.RPAR:
303 prevp = preceding_leaf(p)
304 if not prevp or prevp.type != token.NUMBER:
307 elif t == token.LSQB:
310 elif prev.type != token.COMMA:
313 elif p.type == syms.argument:
319 prevp = preceding_leaf(p)
320 if not prevp or prevp.type == token.LPAR:
323 elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:
326 elif p.type == syms.decorator:
330 elif p.type == syms.dotted_name:
334 prevp = preceding_leaf(p)
335 if not prevp or prevp.type == token.AT or prevp.type == token.DOT:
338 elif p.type == syms.classdef:
342 if prev and prev.type == token.LPAR:
345 elif p.type in {syms.subscript, syms.sliceop}:
348 assert p.parent is not None, "subscripts are always parented"
349 if p.parent.type == syms.subscriptlist:
354 elif not complex_subscript:
357 elif p.type == syms.atom:
358 if prev and t == token.DOT:
359 # dots, but not the first one.
362 elif p.type == syms.dictsetmaker:
364 if prev and prev.type == token.DOUBLESTAR:
367 elif p.type in {syms.factor, syms.star_expr}:
370 prevp = preceding_leaf(p)
371 if not prevp or prevp.type in OPENING_BRACKETS:
374 prevp_parent = prevp.parent
375 assert prevp_parent is not None
376 if prevp.type == token.COLON and prevp_parent.type in {
382 elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:
385 elif t in {token.NAME, token.NUMBER, token.STRING}:
388 elif p.type == syms.import_from:
390 if prev and prev.type == token.DOT:
393 elif t == token.NAME:
397 if prev and prev.type == token.DOT:
400 elif p.type == syms.sliceop:
406 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
407 """Return the first leaf that precedes `node`, if any."""
409 res = node.prev_sibling
411 if isinstance(res, Leaf):
415 return list(res.leaves())[-1]
424 def prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool:
425 """Return if the `node` and its previous siblings match types against the provided
426 list of tokens; the provided `node`has its type matched against the last element in
427 the list. `None` can be used as the first element to declare that the start of the
428 list is anchored at the start of its parent's children."""
431 if tokens[-1] is None:
435 if node.type != tokens[-1]:
437 return prev_siblings_are(node.prev_sibling, tokens[:-1])
440 def last_two_except(leaves: List[Leaf], omit: Collection[LeafID]) -> Tuple[Leaf, Leaf]:
441 """Return (penultimate, last) leaves skipping brackets in `omit` and contents."""
444 for leaf in reversed(leaves):
446 if leaf is stop_after:
454 stop_after = leaf.opening_bracket
458 raise LookupError("Last two leaves were also skipped")
461 def parent_type(node: Optional[LN]) -> Optional[NodeType]:
464 @node.parent.type, if @node is not None and has a parent.
468 if node is None or node.parent is None:
471 return node.parent.type
474 def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
475 """Return the child of `ancestor` that contains `descendant`."""
476 node: Optional[LN] = descendant
477 while node and node.parent != ancestor:
482 def replace_child(old_child: LN, new_child: LN) -> None:
485 * If @old_child.parent is set, replace @old_child with @new_child in
486 @old_child's underlying Node structure.
488 * Otherwise, this function does nothing.
490 parent = old_child.parent
494 child_idx = old_child.remove()
495 if child_idx is not None:
496 parent.insert_child(child_idx, new_child)
499 def container_of(leaf: Leaf) -> LN:
500 """Return `leaf` or one of its ancestors that is the topmost container of it.
502 By "container" we mean a node where `leaf` is the very first child.
504 same_prefix = leaf.prefix
507 parent = container.parent
511 if parent.children[0].prefix != same_prefix:
514 if parent.type == syms.file_input:
517 if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:
524 def first_leaf_column(node: Node) -> Optional[int]:
525 """Returns the column of the first leaf child of a node."""
526 for child in node.children:
527 if isinstance(child, Leaf):
532 def first_child_is_arith(node: Node) -> bool:
533 """Whether first child is an arithmetic or a binary arithmetic expression"""
540 return bool(node.children and node.children[0].type in expr_types)
543 def is_docstring(leaf: Leaf) -> bool:
544 if prev_siblings_are(
545 leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt]
549 # Multiline docstring on the same line as the `def`.
550 if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]):
551 # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python
552 # grammar. We're safe to return True without further checks.
558 def is_empty_tuple(node: LN) -> bool:
559 """Return True if `node` holds an empty tuple."""
561 node.type == syms.atom
562 and len(node.children) == 2
563 and node.children[0].type == token.LPAR
564 and node.children[1].type == token.RPAR
568 def is_one_tuple(node: LN) -> bool:
569 """Return True if `node` holds a tuple with one element, with or without parens."""
570 if node.type == syms.atom:
571 gexp = unwrap_singleton_parenthesis(node)
572 if gexp is None or gexp.type != syms.testlist_gexp:
575 return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
578 node.type in IMPLICIT_TUPLE
579 and len(node.children) == 2
580 and node.children[1].type == token.COMMA
584 def is_one_tuple_between(opening: Leaf, closing: Leaf, leaves: List[Leaf]) -> bool:
585 """Return True if content between `opening` and `closing` looks like a one-tuple."""
586 if opening.type != token.LPAR and closing.type != token.RPAR:
589 depth = closing.bracket_depth + 1
590 for _opening_index, leaf in enumerate(leaves):
595 raise LookupError("Opening paren not found in `leaves`")
599 for leaf in leaves[_opening_index:]:
603 bracket_depth = leaf.bracket_depth
604 if bracket_depth == depth and leaf.type == token.COMMA:
606 if leaf.parent and leaf.parent.type in {
616 def is_walrus_assignment(node: LN) -> bool:
617 """Return True iff `node` is of the shape ( test := test )"""
618 inner = unwrap_singleton_parenthesis(node)
619 return inner is not None and inner.type == syms.namedexpr_test
622 def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool:
623 """Return True iff `node` is a trailer valid in a simple decorator"""
624 return node.type == syms.trailer and (
626 len(node.children) == 2
627 and node.children[0].type == token.DOT
628 and node.children[1].type == token.NAME
630 # last trailer can be an argument-less parentheses pair
633 and len(node.children) == 2
634 and node.children[0].type == token.LPAR
635 and node.children[1].type == token.RPAR
637 # last trailer can be arguments
640 and len(node.children) == 3
641 and node.children[0].type == token.LPAR
642 # and node.children[1].type == syms.argument
643 and node.children[2].type == token.RPAR
648 def is_simple_decorator_expression(node: LN) -> bool:
649 """Return True iff `node` could be a 'dotted name' decorator
651 This function takes the node of the 'namedexpr_test' of the new decorator
652 grammar and test if it would be valid under the old decorator grammar.
654 The old grammar was: decorator: @ dotted_name [arguments] NEWLINE
655 The new grammar is : decorator: @ namedexpr_test NEWLINE
657 if node.type == token.NAME:
659 if node.type == syms.power:
662 node.children[0].type == token.NAME
663 and all(map(is_simple_decorator_trailer, node.children[1:-1]))
665 len(node.children) < 2
666 or is_simple_decorator_trailer(node.children[-1], last=True)
672 def is_yield(node: LN) -> bool:
673 """Return True if `node` holds a `yield` or `yield from` expression."""
674 if node.type == syms.yield_expr:
677 if node.type == token.NAME and node.value == "yield": # type: ignore
680 if node.type != syms.atom:
683 if len(node.children) != 3:
686 lpar, expr, rpar = node.children
687 if lpar.type == token.LPAR and rpar.type == token.RPAR:
688 return is_yield(expr)
693 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
694 """Return True if `leaf` is a star or double star in a vararg or kwarg.
696 If `within` includes VARARGS_PARENTS, this applies to function signatures.
697 If `within` includes UNPACKING_PARENTS, it applies to right hand-side
698 extended iterable unpacking (PEP 3132) and additional unpacking
699 generalizations (PEP 448).
701 if leaf.type not in VARARGS_SPECIALS or not leaf.parent:
705 if p.type == syms.star_expr:
706 # Star expressions are also used as assignment targets in extended
707 # iterable unpacking (PEP 3132). See what its parent is instead.
713 return p.type in within
716 def is_multiline_string(leaf: Leaf) -> bool:
717 """Return True if `leaf` is a multiline string that actually spans many lines."""
718 return has_triple_quotes(leaf.value) and "\n" in leaf.value
721 def is_stub_suite(node: Node) -> bool:
722 """Return True if `node` is a suite with a stub body."""
724 len(node.children) != 4
725 or node.children[0].type != token.NEWLINE
726 or node.children[1].type != token.INDENT
727 or node.children[3].type != token.DEDENT
731 return is_stub_body(node.children[2])
734 def is_stub_body(node: LN) -> bool:
735 """Return True if `node` is a simple statement containing an ellipsis."""
736 if not isinstance(node, Node) or node.type != syms.simple_stmt:
739 if len(node.children) != 2:
742 child = node.children[0]
744 child.type == syms.atom
745 and len(child.children) == 3
746 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
750 def is_atom_with_invisible_parens(node: LN) -> bool:
751 """Given a `LN`, determines whether it's an atom `node` with invisible
752 parens. Useful in dedupe-ing and normalizing parens.
754 if isinstance(node, Leaf) or node.type != syms.atom:
757 first, last = node.children[0], node.children[-1]
759 isinstance(first, Leaf)
760 and first.type == token.LPAR
761 and first.value == ""
762 and isinstance(last, Leaf)
763 and last.type == token.RPAR
768 def is_empty_par(leaf: Leaf) -> bool:
769 return is_empty_lpar(leaf) or is_empty_rpar(leaf)
772 def is_empty_lpar(leaf: Leaf) -> bool:
773 return leaf.type == token.LPAR and leaf.value == ""
776 def is_empty_rpar(leaf: Leaf) -> bool:
777 return leaf.type == token.RPAR and leaf.value == ""
780 def is_import(leaf: Leaf) -> bool:
781 """Return True if the given leaf starts an import statement."""
788 (v == "import" and p and p.type == syms.import_name)
789 or (v == "from" and p and p.type == syms.import_from)
794 def is_type_comment(leaf: Leaf, suffix: str = "") -> bool:
795 """Return True if the given leaf is a special comment.
796 Only returns true for type comments for now."""
799 return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix)
802 def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None:
803 """Wrap `child` in parentheses.
805 This replaces `child` with an atom holding the parentheses and the old
806 child. That requires moving the prefix.
808 If `visible` is False, the leaves will be valueless (and thus invisible).
810 lpar = Leaf(token.LPAR, "(" if visible else "")
811 rpar = Leaf(token.RPAR, ")" if visible else "")
812 prefix = child.prefix
814 index = child.remove() or 0
815 new_child = Node(syms.atom, [lpar, child, rpar])
816 new_child.prefix = prefix
817 parent.insert_child(index, new_child)
820 def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]:
821 """Returns `wrapped` if `node` is of the shape ( wrapped ).
823 Parenthesis can be optional. Returns None otherwise"""
824 if len(node.children) != 3:
827 lpar, wrapped, rpar = node.children
828 if not (lpar.type == token.LPAR and rpar.type == token.RPAR):
834 def ensure_visible(leaf: Leaf) -> None:
835 """Make sure parentheses are visible.
837 They could be invisible as part of some statements (see
838 :func:`normalize_invisible_parens` and :func:`visit_import_from`).
840 if leaf.type == token.LPAR:
842 elif leaf.type == token.RPAR: