All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
3 from dataclasses import dataclass, field
16 from black.brackets import DOT_PRIORITY, BracketTracker
17 from black.mode import Mode, Preview
18 from black.nodes import (
27 is_one_sequence_between,
33 from blib2to3.pgen2 import token
34 from blib2to3.pytree import Leaf, Node
44 """Holds leaves and comments. Can be printed with `str(line)`."""
48 leaves: List[Leaf] = field(default_factory=list)
49 # keys ordered like `leaves`
50 comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict)
51 bracket_tracker: BracketTracker = field(default_factory=BracketTracker)
52 inside_brackets: bool = False
53 should_split_rhs: bool = False
54 magic_trailing_comma: Optional[Leaf] = None
57 self, leaf: Leaf, preformatted: bool = False, track_bracket: bool = False
59 """Add a new `leaf` to the end of the line.
61 Unless `preformatted` is True, the `leaf` will receive a new consistent
62 whitespace prefix and metadata applied by :class:`BracketTracker`.
63 Trailing commas are maybe removed, unpacked for loop variables are
64 demoted from being delimiters.
66 Inline comments are put aside.
68 has_value = leaf.type in BRACKETS or bool(leaf.value.strip())
72 if token.COLON == leaf.type and self.is_class_paren_empty:
74 if self.leaves and not preformatted:
75 # Note: at this point leaf.prefix should be empty except for
76 # imports, for which we only preserve newlines.
77 leaf.prefix += whitespace(
78 leaf, complex_subscript=self.is_complex_subscript(leaf)
80 if self.inside_brackets or not preformatted or track_bracket:
81 self.bracket_tracker.mark(leaf)
82 if self.mode.magic_trailing_comma:
83 if self.has_magic_trailing_comma(leaf):
84 self.magic_trailing_comma = leaf
85 elif self.has_magic_trailing_comma(leaf, ensure_removable=True):
86 self.remove_trailing_comma()
87 if not self.append_comment(leaf):
88 self.leaves.append(leaf)
90 def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
91 """Like :func:`append()` but disallow invalid standalone comment structure.
93 Raises ValueError when any `leaf` is appended after a standalone comment
94 or when a standalone comment is not the first leaf on the line.
96 if self.bracket_tracker.depth == 0:
98 raise ValueError("cannot append to standalone comments")
100 if self.leaves and leaf.type == STANDALONE_COMMENT:
102 "cannot append standalone comments to a populated line"
105 self.append(leaf, preformatted=preformatted)
108 def is_comment(self) -> bool:
109 """Is this line a standalone comment?"""
110 return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT
113 def is_decorator(self) -> bool:
114 """Is this line a decorator?"""
115 return bool(self) and self.leaves[0].type == token.AT
118 def is_import(self) -> bool:
119 """Is this an import line?"""
120 return bool(self) and is_import(self.leaves[0])
123 def is_class(self) -> bool:
124 """Is this line a class definition?"""
127 and self.leaves[0].type == token.NAME
128 and self.leaves[0].value == "class"
132 def is_stub_class(self) -> bool:
133 """Is this line a class definition with a body consisting only of "..."?"""
134 return self.is_class and self.leaves[-3:] == [
135 Leaf(token.DOT, ".") for _ in range(3)
139 def is_def(self) -> bool:
140 """Is this a function definition? (Also returns True for async defs.)"""
142 first_leaf = self.leaves[0]
147 second_leaf: Optional[Leaf] = self.leaves[1]
150 return (first_leaf.type == token.NAME and first_leaf.value == "def") or (
151 first_leaf.type == token.ASYNC
152 and second_leaf is not None
153 and second_leaf.type == token.NAME
154 and second_leaf.value == "def"
158 def is_class_paren_empty(self) -> bool:
159 """Is this a class with no base classes but using parentheses?
161 Those are unnecessary and should be removed.
165 and len(self.leaves) == 4
167 and self.leaves[2].type == token.LPAR
168 and self.leaves[2].value == "("
169 and self.leaves[3].type == token.RPAR
170 and self.leaves[3].value == ")"
174 def is_triple_quoted_string(self) -> bool:
175 """Is the line a triple quoted string?"""
178 and self.leaves[0].type == token.STRING
179 and self.leaves[0].value.startswith(('"""', "'''"))
183 def opens_block(self) -> bool:
184 """Does this line open a new level of indentation."""
185 if len(self.leaves) == 0:
187 return self.leaves[-1].type == token.COLON
189 def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
190 """If so, needs to be split before emitting."""
191 for leaf in self.leaves:
192 if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit:
197 def contains_uncollapsable_type_comments(self) -> bool:
200 last_leaf = self.leaves[-1]
201 ignored_ids.add(id(last_leaf))
202 if last_leaf.type == token.COMMA or (
203 last_leaf.type == token.RPAR and not last_leaf.value
205 # When trailing commas or optional parens are inserted by Black for
206 # consistency, comments after the previous last element are not moved
207 # (they don't have to, rendering will still be correct). So we ignore
208 # trailing commas and invisible.
209 last_leaf = self.leaves[-2]
210 ignored_ids.add(id(last_leaf))
214 # A type comment is uncollapsable if it is attached to a leaf
215 # that isn't at the end of the line (since that could cause it
216 # to get associated to a different argument) or if there are
217 # comments before it (since that could cause it to get hidden
220 for leaf_id, comments in self.comments.items():
221 for comment in comments:
222 if is_type_comment(comment):
224 not is_type_comment(comment, " ignore")
225 and leaf_id not in ignored_ids
233 def contains_unsplittable_type_ignore(self) -> bool:
237 # If a 'type: ignore' is attached to the end of a line, we
238 # can't split the line, because we can't know which of the
239 # subexpressions the ignore was meant to apply to.
241 # We only want this to apply to actual physical lines from the
242 # original source, though: we don't want the presence of a
243 # 'type: ignore' at the end of a multiline expression to
244 # justify pushing it all onto one line. Thus we
245 # (unfortunately) need to check the actual source lines and
246 # only report an unsplittable 'type: ignore' if this line was
247 # one line in the original code.
249 # Grab the first and last line numbers, skipping generated leaves
250 first_line = next((leaf.lineno for leaf in self.leaves if leaf.lineno != 0), 0)
252 (leaf.lineno for leaf in reversed(self.leaves) if leaf.lineno != 0), 0
255 if first_line == last_line:
256 # We look at the last two leaves since a comma or an
257 # invisible paren could have been added at the end of the
259 for node in self.leaves[-2:]:
260 for comment in self.comments.get(id(node), []):
261 if is_type_comment(comment, " ignore"):
266 def contains_multiline_strings(self) -> bool:
267 return any(is_multiline_string(leaf) for leaf in self.leaves)
269 def has_magic_trailing_comma(
270 self, closing: Leaf, ensure_removable: bool = False
272 """Return True if we have a magic trailing comma, that is when:
273 - there's a trailing comma here
274 - it's not a one-tuple
275 - it's not a single-element subscript
276 Additionally, if ensure_removable:
277 - it's not from square bracket indexing
278 (specifically, single-element square bracket indexing with
279 Preview.skip_magic_trailing_comma_in_subscript)
282 closing.type in CLOSING_BRACKETS
284 and self.leaves[-1].type == token.COMMA
288 if closing.type == token.RBRACE:
291 if closing.type == token.RSQB:
293 Preview.one_element_subscript in self.mode
295 and closing.parent.type == syms.trailer
296 and closing.opening_bracket
297 and is_one_sequence_between(
298 closing.opening_bracket,
301 brackets=(token.LSQB, token.RSQB),
306 if not ensure_removable:
309 comma = self.leaves[-1]
310 if comma.parent is None:
312 if Preview.skip_magic_trailing_comma_in_subscript in self.mode:
314 comma.parent.type != syms.subscriptlist
315 or closing.opening_bracket is None
316 or not is_one_sequence_between(
317 closing.opening_bracket,
320 brackets=(token.LSQB, token.RSQB),
323 return comma.parent.type == syms.listmaker
328 if closing.opening_bracket is not None and not is_one_sequence_between(
329 closing.opening_bracket, closing, self.leaves
335 def append_comment(self, comment: Leaf) -> bool:
336 """Add an inline or standalone comment to the line."""
338 comment.type == STANDALONE_COMMENT
339 and self.bracket_tracker.any_open_brackets()
344 if comment.type != token.COMMENT:
348 comment.type = STANDALONE_COMMENT
352 last_leaf = self.leaves[-1]
354 last_leaf.type == token.RPAR
355 and not last_leaf.value
357 and len(list(last_leaf.parent.leaves())) <= 3
358 and not is_type_comment(comment)
360 # Comments on an optional parens wrapping a single leaf should belong to
361 # the wrapped node except if it's a type comment. Pinning the comment like
362 # this avoids unstable formatting caused by comment migration.
363 if len(self.leaves) < 2:
364 comment.type = STANDALONE_COMMENT
368 last_leaf = self.leaves[-2]
369 self.comments.setdefault(id(last_leaf), []).append(comment)
372 def comments_after(self, leaf: Leaf) -> List[Leaf]:
373 """Generate comments that should appear directly after `leaf`."""
374 return self.comments.get(id(leaf), [])
376 def remove_trailing_comma(self) -> None:
377 """Remove the trailing comma and moves the comments attached to it."""
378 trailing_comma = self.leaves.pop()
379 trailing_comma_comments = self.comments.pop(id(trailing_comma), [])
380 self.comments.setdefault(id(self.leaves[-1]), []).extend(
381 trailing_comma_comments
384 def is_complex_subscript(self, leaf: Leaf) -> bool:
385 """Return True iff `leaf` is part of a slice with non-trivial exprs."""
386 open_lsqb = self.bracket_tracker.get_open_lsqb()
387 if open_lsqb is None:
390 subscript_start = open_lsqb.next_sibling
392 if isinstance(subscript_start, Node):
393 if subscript_start.type == syms.listmaker:
396 if subscript_start.type == syms.subscriptlist:
397 subscript_start = child_towards(subscript_start, leaf)
398 return subscript_start is not None and any(
399 n.type in TEST_DESCENDANTS for n in subscript_start.pre_order()
402 def enumerate_with_length(
403 self, reversed: bool = False
404 ) -> Iterator[Tuple[Index, Leaf, int]]:
405 """Return an enumeration of leaves with their length.
407 Stops prematurely on multiline strings and standalone comments.
410 Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]],
411 enumerate_reversed if reversed else enumerate,
413 for index, leaf in op(self.leaves):
414 length = len(leaf.prefix) + len(leaf.value)
415 if "\n" in leaf.value:
416 return # Multiline strings, we can't continue.
418 for comment in self.comments_after(leaf):
419 length += len(comment.value)
421 yield index, leaf, length
423 def clone(self) -> "Line":
427 inside_brackets=self.inside_brackets,
428 should_split_rhs=self.should_split_rhs,
429 magic_trailing_comma=self.magic_trailing_comma,
432 def __str__(self) -> str:
433 """Render the line."""
437 indent = " " * self.depth
438 leaves = iter(self.leaves)
440 res = f"{first.prefix}{indent}{first.value}"
443 for comment in itertools.chain.from_iterable(self.comments.values()):
448 def __bool__(self) -> bool:
449 """Return True if the line has leaves or comments."""
450 return bool(self.leaves or self.comments)
455 """Class that holds information about a block of formatted lines.
457 This is introduced so that the EmptyLineTracker can look behind the standalone
458 comments and adjust their empty lines for class or def lines.
462 previous_block: Optional["LinesBlock"]
465 content_lines: List[str] = field(default_factory=list)
468 def all_lines(self) -> List[str]:
469 empty_line = str(Line(mode=self.mode))
471 [empty_line * self.before] + self.content_lines + [empty_line * self.after]
476 class EmptyLineTracker:
477 """Provides a stateful method that returns the number of potential extra
478 empty lines needed before and after the currently processed line.
480 Note: this tracker works on lines that haven't been split yet. It assumes
481 the prefix of the first leaf consists of optional newlines. Those newlines
482 are consumed by `maybe_empty_lines()` and included in the computation.
486 previous_line: Optional[Line] = None
487 previous_block: Optional[LinesBlock] = None
488 previous_defs: List[int] = field(default_factory=list)
489 semantic_leading_comment: Optional[LinesBlock] = None
491 def maybe_empty_lines(self, current_line: Line) -> LinesBlock:
492 """Return the number of extra empty lines before and after the `current_line`.
494 This is for separating `def`, `async def` and `class` with extra empty
495 lines (two on module-level).
497 before, after = self._maybe_empty_lines(current_line)
498 previous_after = self.previous_block.after if self.previous_block else 0
500 # Black should not insert empty lines at the beginning
503 if self.previous_line is None
504 else before - previous_after
508 previous_block=self.previous_block,
509 original_line=current_line,
514 # Maintain the semantic_leading_comment state.
515 if current_line.is_comment:
516 if self.previous_line is None or (
517 not self.previous_line.is_decorator
518 # `or before` means this comment already has an empty line before
519 and (not self.previous_line.is_comment or before)
520 and (self.semantic_leading_comment is None or before)
522 self.semantic_leading_comment = block
523 elif not current_line.is_decorator:
524 self.semantic_leading_comment = None
526 self.previous_line = current_line
527 self.previous_block = block
530 def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
532 if current_line.depth == 0:
533 max_allowed = 1 if self.mode.is_pyi else 2
534 if current_line.leaves:
535 # Consume the first leaf's extra newlines.
536 first_leaf = current_line.leaves[0]
537 before = first_leaf.prefix.count("\n")
538 before = min(before, max_allowed)
539 first_leaf.prefix = ""
542 depth = current_line.depth
543 while self.previous_defs and self.previous_defs[-1] >= depth:
545 assert self.previous_line is not None
546 if depth and not current_line.is_def and self.previous_line.is_def:
547 # Empty lines between attributes and methods should be preserved.
548 before = min(1, before)
558 and self.previous_defs[-1]
559 and current_line.leaves[-1].type == token.COLON
561 current_line.leaves[0].value
562 not in ("with", "try", "for", "while", "if", "match")
565 # We shouldn't add two newlines between an indented function and
566 # a dependent non-indented clause. This is to avoid issues with
567 # conditional function definitions that are technically top-level
568 # and therefore get two trailing newlines, but look weird and
569 # inconsistent when they're followed by elif, else, etc. This is
570 # worse because these functions only get *one* preceding newline
575 self.previous_defs.pop()
576 if current_line.is_decorator or current_line.is_def or current_line.is_class:
577 return self._maybe_empty_lines_for_class_or_def(current_line, before)
581 and self.previous_line.is_import
582 and not current_line.is_import
583 and depth == self.previous_line.depth
585 return (before or 1), 0
589 and self.previous_line.is_class
590 and current_line.is_triple_quoted_string
595 Preview.remove_block_trailing_newline in current_line.mode
596 and self.previous_line
597 and self.previous_line.opens_block
602 def _maybe_empty_lines_for_class_or_def(
603 self, current_line: Line, before: int
604 ) -> Tuple[int, int]:
605 if not current_line.is_decorator:
606 self.previous_defs.append(current_line.depth)
607 if self.previous_line is None:
608 # Don't insert empty lines before the first line in the file.
611 if self.previous_line.is_decorator:
612 if self.mode.is_pyi and current_line.is_stub_class:
613 # Insert an empty line after a decorated stub class
618 if self.previous_line.depth < current_line.depth and (
619 self.previous_line.is_class or self.previous_line.is_def
623 comment_to_add_newlines: Optional[LinesBlock] = None
625 self.previous_line.is_comment
626 and self.previous_line.depth == current_line.depth
629 slc = self.semantic_leading_comment
631 Preview.empty_lines_before_class_or_def_with_leading_comments
634 and slc.previous_block is not None
635 and not slc.previous_block.original_line.is_class
636 and not slc.previous_block.original_line.opens_block
639 comment_to_add_newlines = slc
644 if current_line.is_class or self.previous_line.is_class:
645 if self.previous_line.depth < current_line.depth:
647 elif self.previous_line.depth > current_line.depth:
649 elif current_line.is_stub_class and self.previous_line.is_stub_class:
650 # No blank line between classes with an empty body
655 current_line.is_def or current_line.is_decorator
656 ) and not self.previous_line.is_def:
657 if current_line.depth:
658 # In classes empty lines between attributes and methods should
660 newlines = min(1, before)
662 # Blank line between a block of functions (maybe with preceding
663 # decorators) and a block of non-functions
665 elif self.previous_line.depth > current_line.depth:
670 newlines = 1 if current_line.depth else 2
671 if comment_to_add_newlines is not None:
672 previous_block = comment_to_add_newlines.previous_block
673 if previous_block is not None:
674 comment_to_add_newlines.before = (
675 max(comment_to_add_newlines.before, newlines) - previous_block.after
681 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
682 """Like `reversed(enumerate(sequence))` if that were possible."""
683 index = len(sequence) - 1
684 for element in reversed(sequence):
685 yield (index, element)
690 new_line: Line, old_line: Line, leaves: List[Leaf], preformatted: bool = False
693 Append leaves (taken from @old_line) to @new_line, making sure to fix the
694 underlying Node structure where appropriate.
696 All of the leaves in @leaves are duplicated. The duplicates are then
697 appended to @new_line and used to replace their originals in the underlying
698 Node structure. Any comments attached to the old leaves are reattached to
702 set(@leaves) is a subset of set(@old_line.leaves).
704 for old_leaf in leaves:
705 new_leaf = Leaf(old_leaf.type, old_leaf.value)
706 replace_child(old_leaf, new_leaf)
707 new_line.append(new_leaf, preformatted=preformatted)
709 for comment_leaf in old_line.comments_after(old_leaf):
710 new_line.append(comment_leaf, preformatted=True)
713 def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
714 """Return True if `line` is no longer than `line_length`.
716 Uses the provided `line_str` rendering, if any, otherwise computes a new one.
719 line_str = line_to_string(line)
721 len(line_str) <= line_length
722 and "\n" not in line_str # multiline strings
723 and not line.contains_standalone_comments()
727 def can_be_split(line: Line) -> bool:
728 """Return False if the line cannot be split *for sure*.
730 This is not an exhaustive search but a cheap heuristic that we can use to
731 avoid some unfortunate formattings (mostly around wrapping unsplittable code
732 in unnecessary parentheses).
738 if leaves[0].type == token.STRING and leaves[1].type == token.DOT:
742 for leaf in leaves[-2::-1]:
743 if leaf.type in OPENING_BRACKETS:
744 if next.type not in CLOSING_BRACKETS:
748 elif leaf.type == token.DOT:
750 elif leaf.type == token.NAME:
751 if not (next.type == token.DOT or next.type in OPENING_BRACKETS):
754 elif leaf.type not in CLOSING_BRACKETS:
757 if dot_count > 1 and call_count > 1:
763 def can_omit_invisible_parens(
767 """Does `line` have a shape safe to reformat without optional parens around it?
769 Returns True for only a subset of potentially nice looking formattings but
770 the point is to not return false positives that end up producing lines that
773 bt = line.bracket_tracker
774 if not bt.delimiters:
775 # Without delimiters the optional parentheses are useless.
778 max_priority = bt.max_delimiter_priority()
779 if bt.delimiter_count_with_priority(max_priority) > 1:
780 # With more than one delimiter of a kind the optional parentheses read better.
783 if max_priority == DOT_PRIORITY:
784 # A single stranded method call doesn't require optional parentheses.
787 assert len(line.leaves) >= 2, "Stranded delimiter"
789 # With a single delimiter, omit if the expression starts or ends with
791 first = line.leaves[0]
792 second = line.leaves[1]
793 if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS:
794 if _can_omit_opening_paren(line, first=first, line_length=line_length):
797 # Note: we are not returning False here because a line might have *both*
798 # a leading opening bracket and a trailing closing bracket. If the
799 # opening bracket doesn't match our rule, maybe the closing will.
801 penultimate = line.leaves[-2]
802 last = line.leaves[-1]
805 last.type == token.RPAR
806 or last.type == token.RBRACE
808 # don't use indexing for omitting optional parentheses;
810 last.type == token.RSQB
812 and last.parent.type != syms.trailer
815 if penultimate.type in OPENING_BRACKETS:
816 # Empty brackets don't help.
819 if is_multiline_string(first):
820 # Additional wrapping of a multiline string in this situation is
824 if _can_omit_closing_paren(line, last=last, line_length=line_length):
830 def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool:
831 """See `can_omit_invisible_parens`."""
833 length = 4 * line.depth
835 for _index, leaf, leaf_length in line.enumerate_with_length():
836 if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first:
839 length += leaf_length
840 if length > line_length:
843 if leaf.type in OPENING_BRACKETS:
844 # There are brackets we can further split on.
848 # checked the entire string and line length wasn't exceeded
849 if len(line.leaves) == _index + 1:
855 def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool:
856 """See `can_omit_invisible_parens`."""
857 length = 4 * line.depth
858 seen_other_brackets = False
859 for _index, leaf, leaf_length in line.enumerate_with_length():
860 length += leaf_length
861 if leaf is last.opening_bracket:
862 if seen_other_brackets or length <= line_length:
865 elif leaf.type in OPENING_BRACKETS:
866 # There are brackets we can further split on.
867 seen_other_brackets = True
872 def line_to_string(line: Line) -> str:
873 """Returns the string representation of @line.
875 WARNING: This is known to be computationally expensive.
877 return str(line).strip("\n")