All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
3 from dataclasses import dataclass, field
16 from black.brackets import DOT_PRIORITY, BracketTracker
17 from black.mode import Mode, Preview
18 from black.nodes import (
27 is_one_sequence_between,
33 from blib2to3.pgen2 import token
34 from blib2to3.pytree import Leaf, Node
44 """Holds leaves and comments. Can be printed with `str(line)`."""
48 leaves: List[Leaf] = field(default_factory=list)
49 # keys ordered like `leaves`
50 comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict)
51 bracket_tracker: BracketTracker = field(default_factory=BracketTracker)
52 inside_brackets: bool = False
53 should_split_rhs: bool = False
54 magic_trailing_comma: Optional[Leaf] = None
56 def append(self, leaf: Leaf, preformatted: bool = False) -> None:
57 """Add a new `leaf` to the end of the line.
59 Unless `preformatted` is True, the `leaf` will receive a new consistent
60 whitespace prefix and metadata applied by :class:`BracketTracker`.
61 Trailing commas are maybe removed, unpacked for loop variables are
62 demoted from being delimiters.
64 Inline comments are put aside.
66 has_value = leaf.type in BRACKETS or bool(leaf.value.strip())
70 if token.COLON == leaf.type and self.is_class_paren_empty:
72 if self.leaves and not preformatted:
73 # Note: at this point leaf.prefix should be empty except for
74 # imports, for which we only preserve newlines.
75 leaf.prefix += whitespace(
76 leaf, complex_subscript=self.is_complex_subscript(leaf)
78 if self.inside_brackets or not preformatted:
79 self.bracket_tracker.mark(leaf)
80 if self.mode.magic_trailing_comma:
81 if self.has_magic_trailing_comma(leaf):
82 self.magic_trailing_comma = leaf
83 elif self.has_magic_trailing_comma(leaf, ensure_removable=True):
84 self.remove_trailing_comma()
85 if not self.append_comment(leaf):
86 self.leaves.append(leaf)
88 def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
89 """Like :func:`append()` but disallow invalid standalone comment structure.
91 Raises ValueError when any `leaf` is appended after a standalone comment
92 or when a standalone comment is not the first leaf on the line.
94 if self.bracket_tracker.depth == 0:
96 raise ValueError("cannot append to standalone comments")
98 if self.leaves and leaf.type == STANDALONE_COMMENT:
100 "cannot append standalone comments to a populated line"
103 self.append(leaf, preformatted=preformatted)
106 def is_comment(self) -> bool:
107 """Is this line a standalone comment?"""
108 return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT
111 def is_decorator(self) -> bool:
112 """Is this line a decorator?"""
113 return bool(self) and self.leaves[0].type == token.AT
116 def is_import(self) -> bool:
117 """Is this an import line?"""
118 return bool(self) and is_import(self.leaves[0])
121 def is_class(self) -> bool:
122 """Is this line a class definition?"""
125 and self.leaves[0].type == token.NAME
126 and self.leaves[0].value == "class"
130 def is_stub_class(self) -> bool:
131 """Is this line a class definition with a body consisting only of "..."?"""
132 return self.is_class and self.leaves[-3:] == [
133 Leaf(token.DOT, ".") for _ in range(3)
137 def is_def(self) -> bool:
138 """Is this a function definition? (Also returns True for async defs.)"""
140 first_leaf = self.leaves[0]
145 second_leaf: Optional[Leaf] = self.leaves[1]
148 return (first_leaf.type == token.NAME and first_leaf.value == "def") or (
149 first_leaf.type == token.ASYNC
150 and second_leaf is not None
151 and second_leaf.type == token.NAME
152 and second_leaf.value == "def"
156 def is_class_paren_empty(self) -> bool:
157 """Is this a class with no base classes but using parentheses?
159 Those are unnecessary and should be removed.
163 and len(self.leaves) == 4
165 and self.leaves[2].type == token.LPAR
166 and self.leaves[2].value == "("
167 and self.leaves[3].type == token.RPAR
168 and self.leaves[3].value == ")"
172 def is_triple_quoted_string(self) -> bool:
173 """Is the line a triple quoted string?"""
176 and self.leaves[0].type == token.STRING
177 and self.leaves[0].value.startswith(('"""', "'''"))
181 def opens_block(self) -> bool:
182 """Does this line open a new level of indentation."""
183 if len(self.leaves) == 0:
185 return self.leaves[-1].type == token.COLON
187 def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
188 """If so, needs to be split before emitting."""
189 for leaf in self.leaves:
190 if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit:
195 def contains_uncollapsable_type_comments(self) -> bool:
198 last_leaf = self.leaves[-1]
199 ignored_ids.add(id(last_leaf))
200 if last_leaf.type == token.COMMA or (
201 last_leaf.type == token.RPAR and not last_leaf.value
203 # When trailing commas or optional parens are inserted by Black for
204 # consistency, comments after the previous last element are not moved
205 # (they don't have to, rendering will still be correct). So we ignore
206 # trailing commas and invisible.
207 last_leaf = self.leaves[-2]
208 ignored_ids.add(id(last_leaf))
212 # A type comment is uncollapsable if it is attached to a leaf
213 # that isn't at the end of the line (since that could cause it
214 # to get associated to a different argument) or if there are
215 # comments before it (since that could cause it to get hidden
218 for leaf_id, comments in self.comments.items():
219 for comment in comments:
220 if is_type_comment(comment):
222 not is_type_comment(comment, " ignore")
223 and leaf_id not in ignored_ids
231 def contains_unsplittable_type_ignore(self) -> bool:
235 # If a 'type: ignore' is attached to the end of a line, we
236 # can't split the line, because we can't know which of the
237 # subexpressions the ignore was meant to apply to.
239 # We only want this to apply to actual physical lines from the
240 # original source, though: we don't want the presence of a
241 # 'type: ignore' at the end of a multiline expression to
242 # justify pushing it all onto one line. Thus we
243 # (unfortunately) need to check the actual source lines and
244 # only report an unsplittable 'type: ignore' if this line was
245 # one line in the original code.
247 # Grab the first and last line numbers, skipping generated leaves
248 first_line = next((leaf.lineno for leaf in self.leaves if leaf.lineno != 0), 0)
250 (leaf.lineno for leaf in reversed(self.leaves) if leaf.lineno != 0), 0
253 if first_line == last_line:
254 # We look at the last two leaves since a comma or an
255 # invisible paren could have been added at the end of the
257 for node in self.leaves[-2:]:
258 for comment in self.comments.get(id(node), []):
259 if is_type_comment(comment, " ignore"):
264 def contains_multiline_strings(self) -> bool:
265 return any(is_multiline_string(leaf) for leaf in self.leaves)
267 def has_magic_trailing_comma(
268 self, closing: Leaf, ensure_removable: bool = False
270 """Return True if we have a magic trailing comma, that is when:
271 - there's a trailing comma here
272 - it's not a one-tuple
273 - it's not a single-element subscript
274 Additionally, if ensure_removable:
275 - it's not from square bracket indexing
276 (specifically, single-element square bracket indexing with
277 Preview.skip_magic_trailing_comma_in_subscript)
280 closing.type in CLOSING_BRACKETS
282 and self.leaves[-1].type == token.COMMA
286 if closing.type == token.RBRACE:
289 if closing.type == token.RSQB:
291 Preview.one_element_subscript in self.mode
293 and closing.parent.type == syms.trailer
294 and closing.opening_bracket
295 and is_one_sequence_between(
296 closing.opening_bracket,
299 brackets=(token.LSQB, token.RSQB),
304 if not ensure_removable:
307 comma = self.leaves[-1]
308 if comma.parent is None:
310 if Preview.skip_magic_trailing_comma_in_subscript in self.mode:
312 comma.parent.type != syms.subscriptlist
313 or closing.opening_bracket is None
314 or not is_one_sequence_between(
315 closing.opening_bracket,
318 brackets=(token.LSQB, token.RSQB),
321 return comma.parent.type == syms.listmaker
326 if closing.opening_bracket is not None and not is_one_sequence_between(
327 closing.opening_bracket, closing, self.leaves
333 def append_comment(self, comment: Leaf) -> bool:
334 """Add an inline or standalone comment to the line."""
336 comment.type == STANDALONE_COMMENT
337 and self.bracket_tracker.any_open_brackets()
342 if comment.type != token.COMMENT:
346 comment.type = STANDALONE_COMMENT
350 last_leaf = self.leaves[-1]
352 last_leaf.type == token.RPAR
353 and not last_leaf.value
355 and len(list(last_leaf.parent.leaves())) <= 3
356 and not is_type_comment(comment)
358 # Comments on an optional parens wrapping a single leaf should belong to
359 # the wrapped node except if it's a type comment. Pinning the comment like
360 # this avoids unstable formatting caused by comment migration.
361 if len(self.leaves) < 2:
362 comment.type = STANDALONE_COMMENT
366 last_leaf = self.leaves[-2]
367 self.comments.setdefault(id(last_leaf), []).append(comment)
370 def comments_after(self, leaf: Leaf) -> List[Leaf]:
371 """Generate comments that should appear directly after `leaf`."""
372 return self.comments.get(id(leaf), [])
374 def remove_trailing_comma(self) -> None:
375 """Remove the trailing comma and moves the comments attached to it."""
376 trailing_comma = self.leaves.pop()
377 trailing_comma_comments = self.comments.pop(id(trailing_comma), [])
378 self.comments.setdefault(id(self.leaves[-1]), []).extend(
379 trailing_comma_comments
382 def is_complex_subscript(self, leaf: Leaf) -> bool:
383 """Return True iff `leaf` is part of a slice with non-trivial exprs."""
384 open_lsqb = self.bracket_tracker.get_open_lsqb()
385 if open_lsqb is None:
388 subscript_start = open_lsqb.next_sibling
390 if isinstance(subscript_start, Node):
391 if subscript_start.type == syms.listmaker:
394 if subscript_start.type == syms.subscriptlist:
395 subscript_start = child_towards(subscript_start, leaf)
396 return subscript_start is not None and any(
397 n.type in TEST_DESCENDANTS for n in subscript_start.pre_order()
400 def enumerate_with_length(
401 self, reversed: bool = False
402 ) -> Iterator[Tuple[Index, Leaf, int]]:
403 """Return an enumeration of leaves with their length.
405 Stops prematurely on multiline strings and standalone comments.
408 Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]],
409 enumerate_reversed if reversed else enumerate,
411 for index, leaf in op(self.leaves):
412 length = len(leaf.prefix) + len(leaf.value)
413 if "\n" in leaf.value:
414 return # Multiline strings, we can't continue.
416 for comment in self.comments_after(leaf):
417 length += len(comment.value)
419 yield index, leaf, length
421 def clone(self) -> "Line":
425 inside_brackets=self.inside_brackets,
426 should_split_rhs=self.should_split_rhs,
427 magic_trailing_comma=self.magic_trailing_comma,
430 def __str__(self) -> str:
431 """Render the line."""
435 indent = " " * self.depth
436 leaves = iter(self.leaves)
438 res = f"{first.prefix}{indent}{first.value}"
441 for comment in itertools.chain.from_iterable(self.comments.values()):
446 def __bool__(self) -> bool:
447 """Return True if the line has leaves or comments."""
448 return bool(self.leaves or self.comments)
452 class EmptyLineTracker:
453 """Provides a stateful method that returns the number of potential extra
454 empty lines needed before and after the currently processed line.
456 Note: this tracker works on lines that haven't been split yet. It assumes
457 the prefix of the first leaf consists of optional newlines. Those newlines
458 are consumed by `maybe_empty_lines()` and included in the computation.
462 previous_line: Optional[Line] = None
463 previous_after: int = 0
464 previous_defs: List[int] = field(default_factory=list)
466 def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
467 """Return the number of extra empty lines before and after the `current_line`.
469 This is for separating `def`, `async def` and `class` with extra empty
470 lines (two on module-level).
472 before, after = self._maybe_empty_lines(current_line)
474 # Black should not insert empty lines at the beginning
477 if self.previous_line is None
478 else before - self.previous_after
480 self.previous_after = after
481 self.previous_line = current_line
484 def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
486 if current_line.depth == 0:
487 max_allowed = 1 if self.is_pyi else 2
488 if current_line.leaves:
489 # Consume the first leaf's extra newlines.
490 first_leaf = current_line.leaves[0]
491 before = first_leaf.prefix.count("\n")
492 before = min(before, max_allowed)
493 first_leaf.prefix = ""
496 depth = current_line.depth
497 while self.previous_defs and self.previous_defs[-1] >= depth:
499 assert self.previous_line is not None
500 if depth and not current_line.is_def and self.previous_line.is_def:
501 # Empty lines between attributes and methods should be preserved.
502 before = min(1, before)
512 and self.previous_defs[-1]
513 and current_line.leaves[-1].type == token.COLON
515 current_line.leaves[0].value
516 not in ("with", "try", "for", "while", "if", "match")
519 # We shouldn't add two newlines between an indented function and
520 # a dependent non-indented clause. This is to avoid issues with
521 # conditional function definitions that are technically top-level
522 # and therefore get two trailing newlines, but look weird and
523 # inconsistent when they're followed by elif, else, etc. This is
524 # worse because these functions only get *one* preceding newline
529 self.previous_defs.pop()
530 if current_line.is_decorator or current_line.is_def or current_line.is_class:
531 return self._maybe_empty_lines_for_class_or_def(current_line, before)
535 and self.previous_line.is_import
536 and not current_line.is_import
537 and depth == self.previous_line.depth
539 return (before or 1), 0
543 and self.previous_line.is_class
544 and current_line.is_triple_quoted_string
549 Preview.remove_block_trailing_newline in current_line.mode
550 and self.previous_line
551 and self.previous_line.opens_block
556 def _maybe_empty_lines_for_class_or_def(
557 self, current_line: Line, before: int
558 ) -> Tuple[int, int]:
559 if not current_line.is_decorator:
560 self.previous_defs.append(current_line.depth)
561 if self.previous_line is None:
562 # Don't insert empty lines before the first line in the file.
565 if self.previous_line.is_decorator:
566 if self.is_pyi and current_line.is_stub_class:
567 # Insert an empty line after a decorated stub class
572 if self.previous_line.depth < current_line.depth and (
573 self.previous_line.is_class or self.previous_line.is_def
578 self.previous_line.is_comment
579 and self.previous_line.depth == current_line.depth
585 if current_line.is_class or self.previous_line.is_class:
586 if self.previous_line.depth < current_line.depth:
588 elif self.previous_line.depth > current_line.depth:
590 elif current_line.is_stub_class and self.previous_line.is_stub_class:
591 # No blank line between classes with an empty body
596 current_line.is_def or current_line.is_decorator
597 ) and not self.previous_line.is_def:
598 if current_line.depth:
599 # In classes empty lines between attributes and methods should
601 newlines = min(1, before)
603 # Blank line between a block of functions (maybe with preceding
604 # decorators) and a block of non-functions
606 elif self.previous_line.depth > current_line.depth:
611 newlines = 1 if current_line.depth else 2
615 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
616 """Like `reversed(enumerate(sequence))` if that were possible."""
617 index = len(sequence) - 1
618 for element in reversed(sequence):
619 yield (index, element)
624 new_line: Line, old_line: Line, leaves: List[Leaf], preformatted: bool = False
627 Append leaves (taken from @old_line) to @new_line, making sure to fix the
628 underlying Node structure where appropriate.
630 All of the leaves in @leaves are duplicated. The duplicates are then
631 appended to @new_line and used to replace their originals in the underlying
632 Node structure. Any comments attached to the old leaves are reattached to
636 set(@leaves) is a subset of set(@old_line.leaves).
638 for old_leaf in leaves:
639 new_leaf = Leaf(old_leaf.type, old_leaf.value)
640 replace_child(old_leaf, new_leaf)
641 new_line.append(new_leaf, preformatted=preformatted)
643 for comment_leaf in old_line.comments_after(old_leaf):
644 new_line.append(comment_leaf, preformatted=True)
647 def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
648 """Return True if `line` is no longer than `line_length`.
650 Uses the provided `line_str` rendering, if any, otherwise computes a new one.
653 line_str = line_to_string(line)
655 len(line_str) <= line_length
656 and "\n" not in line_str # multiline strings
657 and not line.contains_standalone_comments()
661 def can_be_split(line: Line) -> bool:
662 """Return False if the line cannot be split *for sure*.
664 This is not an exhaustive search but a cheap heuristic that we can use to
665 avoid some unfortunate formattings (mostly around wrapping unsplittable code
666 in unnecessary parentheses).
672 if leaves[0].type == token.STRING and leaves[1].type == token.DOT:
676 for leaf in leaves[-2::-1]:
677 if leaf.type in OPENING_BRACKETS:
678 if next.type not in CLOSING_BRACKETS:
682 elif leaf.type == token.DOT:
684 elif leaf.type == token.NAME:
685 if not (next.type == token.DOT or next.type in OPENING_BRACKETS):
688 elif leaf.type not in CLOSING_BRACKETS:
691 if dot_count > 1 and call_count > 1:
697 def can_omit_invisible_parens(
701 """Does `line` have a shape safe to reformat without optional parens around it?
703 Returns True for only a subset of potentially nice looking formattings but
704 the point is to not return false positives that end up producing lines that
707 bt = line.bracket_tracker
708 if not bt.delimiters:
709 # Without delimiters the optional parentheses are useless.
712 max_priority = bt.max_delimiter_priority()
713 if bt.delimiter_count_with_priority(max_priority) > 1:
714 # With more than one delimiter of a kind the optional parentheses read better.
717 if max_priority == DOT_PRIORITY:
718 # A single stranded method call doesn't require optional parentheses.
721 assert len(line.leaves) >= 2, "Stranded delimiter"
723 # With a single delimiter, omit if the expression starts or ends with
725 first = line.leaves[0]
726 second = line.leaves[1]
727 if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS:
728 if _can_omit_opening_paren(line, first=first, line_length=line_length):
731 # Note: we are not returning False here because a line might have *both*
732 # a leading opening bracket and a trailing closing bracket. If the
733 # opening bracket doesn't match our rule, maybe the closing will.
735 penultimate = line.leaves[-2]
736 last = line.leaves[-1]
739 last.type == token.RPAR
740 or last.type == token.RBRACE
742 # don't use indexing for omitting optional parentheses;
744 last.type == token.RSQB
746 and last.parent.type != syms.trailer
749 if penultimate.type in OPENING_BRACKETS:
750 # Empty brackets don't help.
753 if is_multiline_string(first):
754 # Additional wrapping of a multiline string in this situation is
758 if _can_omit_closing_paren(line, last=last, line_length=line_length):
764 def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool:
765 """See `can_omit_invisible_parens`."""
767 length = 4 * line.depth
769 for _index, leaf, leaf_length in line.enumerate_with_length():
770 if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first:
773 length += leaf_length
774 if length > line_length:
777 if leaf.type in OPENING_BRACKETS:
778 # There are brackets we can further split on.
782 # checked the entire string and line length wasn't exceeded
783 if len(line.leaves) == _index + 1:
789 def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool:
790 """See `can_omit_invisible_parens`."""
791 length = 4 * line.depth
792 seen_other_brackets = False
793 for _index, leaf, leaf_length in line.enumerate_with_length():
794 length += leaf_length
795 if leaf is last.opening_bracket:
796 if seen_other_brackets or length <= line_length:
799 elif leaf.type in OPENING_BRACKETS:
800 # There are brackets we can further split on.
801 seen_other_brackets = True
806 def line_to_string(line: Line) -> str:
807 """Returns the string representation of @line.
809 WARNING: This is known to be computationally expensive.
811 return str(line).strip("\n")