All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 Generating lines of code.
5 from functools import partial, wraps
6 from typing import Collection, Iterator, List, Optional, Set, Union, cast
8 from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, max_delimiter_priority_in_atom
9 from black.comments import FMT_OFF, generate_comments, list_comments
10 from black.lines import (
14 can_omit_invisible_parens,
18 from black.mode import Feature, Mode, Preview
19 from black.nodes import (
30 is_atom_with_invisible_parens,
36 is_one_sequence_between,
47 from black.numerics import normalize_numeric_literal
48 from black.strings import (
51 normalize_string_prefix,
52 normalize_string_quotes,
54 from black.trans import (
63 from blib2to3.pgen2 import token
64 from blib2to3.pytree import Leaf, Node
68 LN = Union[Leaf, Node]
71 class CannotSplit(CannotTransform):
72 """A readable split that fits the allotted line length is impossible."""
75 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
76 # See also https://github.com/mypyc/mypyc/issues/827.
77 class LineGenerator(Visitor[Line]):
78 """Generates reformatted Line objects. Empty lines are not emitted.
80 Note: destroys the tree it's visiting by mutating prefixes of its leaves
81 in ways that will no longer stringify to valid Python code on the tree.
84 def __init__(self, mode: Mode) -> None:
86 self.current_line: Line
89 def line(self, indent: int = 0) -> Iterator[Line]:
92 If the line is empty, only emit if it makes sense.
93 If the line is too long, split it first and then generate.
95 If any lines were generated, set up a new current_line.
97 if not self.current_line:
98 self.current_line.depth += indent
99 return # Line is empty, don't emit. Creating a new one unnecessary.
101 complete_line = self.current_line
102 self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
105 def visit_default(self, node: LN) -> Iterator[Line]:
106 """Default `visit_*()` implementation. Recurses to children of `node`."""
107 if isinstance(node, Leaf):
108 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
109 for comment in generate_comments(node, preview=self.mode.preview):
110 if any_open_brackets:
111 # any comment within brackets is subject to splitting
112 self.current_line.append(comment)
113 elif comment.type == token.COMMENT:
114 # regular trailing comment
115 self.current_line.append(comment)
116 yield from self.line()
119 # regular standalone comment
120 yield from self.line()
122 self.current_line.append(comment)
123 yield from self.line()
125 normalize_prefix(node, inside_brackets=any_open_brackets)
126 if self.mode.string_normalization and node.type == token.STRING:
127 node.value = normalize_string_prefix(node.value)
128 node.value = normalize_string_quotes(node.value)
129 if node.type == token.NUMBER:
130 normalize_numeric_literal(node)
131 if node.type not in WHITESPACE:
132 self.current_line.append(node)
133 yield from super().visit_default(node)
135 def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
136 """Increase indentation level, maybe yield a line."""
137 # In blib2to3 INDENT never holds comments.
138 yield from self.line(+1)
139 yield from self.visit_default(node)
141 def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
142 """Decrease indentation level, maybe yield a line."""
143 # The current line might still wait for trailing comments. At DEDENT time
144 # there won't be any (they would be prefixes on the preceding NEWLINE).
145 # Emit the line then.
146 yield from self.line()
148 # While DEDENT has no value, its prefix may contain standalone comments
149 # that belong to the current indentation level. Get 'em.
150 yield from self.visit_default(node)
152 # Finally, emit the dedent.
153 yield from self.line(-1)
156 self, node: Node, keywords: Set[str], parens: Set[str]
158 """Visit a statement.
160 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
161 `def`, `with`, `class`, `assert`, and assignments.
163 The relevant Python language `keywords` for a given statement will be
164 NAME leaves within it. This methods puts those on a separate line.
166 `parens` holds a set of string leaf values immediately after which
167 invisible parens should be put.
169 normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
170 for child in node.children:
171 if is_name_token(child) and child.value in keywords:
172 yield from self.line()
174 yield from self.visit(child)
176 def visit_funcdef(self, node: Node) -> Iterator[Line]:
177 """Visit function definition."""
178 if Preview.annotation_parens not in self.mode:
179 yield from self.visit_stmt(node, keywords={"def"}, parens=set())
181 yield from self.line()
183 # Remove redundant brackets around return type annotation.
184 is_return_annotation = False
185 for child in node.children:
186 if child.type == token.RARROW:
187 is_return_annotation = True
188 elif is_return_annotation:
189 if child.type == syms.atom and child.children[0].type == token.LPAR:
190 if maybe_make_parens_invisible_in_atom(
193 remove_brackets_around_comma=False,
195 wrap_in_parentheses(node, child, visible=False)
197 wrap_in_parentheses(node, child, visible=False)
198 is_return_annotation = False
200 for child in node.children:
201 yield from self.visit(child)
203 def visit_match_case(self, node: Node) -> Iterator[Line]:
204 """Visit either a match or case statement."""
205 normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
207 yield from self.line()
208 for child in node.children:
209 yield from self.visit(child)
211 def visit_suite(self, node: Node) -> Iterator[Line]:
213 if self.mode.is_pyi and is_stub_suite(node):
214 yield from self.visit(node.children[2])
216 yield from self.visit_default(node)
218 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
219 """Visit a statement without nested statements."""
220 prev_type: Optional[int] = None
221 for child in node.children:
222 if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
223 wrap_in_parentheses(node, child, visible=False)
224 prev_type = child.type
226 is_suite_like = node.parent and node.parent.type in STATEMENT
228 if self.mode.is_pyi and is_stub_body(node):
229 yield from self.visit_default(node)
231 yield from self.line(+1)
232 yield from self.visit_default(node)
233 yield from self.line(-1)
239 or not is_stub_suite(node.parent)
241 yield from self.line()
242 yield from self.visit_default(node)
244 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
245 """Visit `async def`, `async for`, `async with`."""
246 yield from self.line()
248 children = iter(node.children)
249 for child in children:
250 yield from self.visit(child)
252 if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
253 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
257 internal_stmt = next(children)
258 for child in internal_stmt.children:
259 yield from self.visit(child)
261 def visit_decorators(self, node: Node) -> Iterator[Line]:
262 """Visit decorators."""
263 for child in node.children:
264 yield from self.line()
265 yield from self.visit(child)
267 def visit_power(self, node: Node) -> Iterator[Line]:
268 for idx, leaf in enumerate(node.children[:-1]):
269 next_leaf = node.children[idx + 1]
271 if not isinstance(leaf, Leaf):
274 value = leaf.value.lower()
276 leaf.type == token.NUMBER
277 and next_leaf.type == syms.trailer
278 # Ensure that we are in an attribute trailer
279 and next_leaf.children[0].type == token.DOT
280 # It shouldn't wrap hexadecimal, binary and octal literals
281 and not value.startswith(("0x", "0b", "0o"))
282 # It shouldn't wrap complex literals
285 wrap_in_parentheses(node, leaf)
287 if Preview.remove_redundant_parens in self.mode:
288 remove_await_parens(node)
290 yield from self.visit_default(node)
292 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
293 """Remove a semicolon and put the other statement on a separate line."""
294 yield from self.line()
296 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
297 """End of file. Process outstanding comments and end with a newline."""
298 yield from self.visit_default(leaf)
299 yield from self.line()
301 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
302 if not self.current_line.bracket_tracker.any_open_brackets():
303 yield from self.line()
304 yield from self.visit_default(leaf)
306 def visit_factor(self, node: Node) -> Iterator[Line]:
307 """Force parentheses between a unary op and a binary power:
311 _operator, operand = node.children
313 operand.type == syms.power
314 and len(operand.children) == 3
315 and operand.children[1].type == token.DOUBLESTAR
317 lpar = Leaf(token.LPAR, "(")
318 rpar = Leaf(token.RPAR, ")")
319 index = operand.remove() or 0
320 node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
321 yield from self.visit_default(node)
323 def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
324 if is_docstring(leaf) and "\\\n" not in leaf.value:
325 # We're ignoring docstrings with backslash newline escapes because changing
326 # indentation of those changes the AST representation of the code.
327 if Preview.normalize_docstring_quotes_and_prefixes_properly in self.mode:
328 # There was a bug where --skip-string-normalization wouldn't stop us
329 # from normalizing docstring prefixes. To maintain stability, we can
330 # only address this buggy behaviour while the preview style is enabled.
331 if self.mode.string_normalization:
332 docstring = normalize_string_prefix(leaf.value)
333 # visit_default() does handle string normalization for us, but
334 # since this method acts differently depending on quote style (ex.
335 # see padding logic below), there's a possibility for unstable
336 # formatting as visit_default() is called *after*. To avoid a
337 # situation where this function formats a docstring differently on
338 # the second pass, normalize it early.
339 docstring = normalize_string_quotes(docstring)
341 docstring = leaf.value
343 # ... otherwise, we'll keep the buggy behaviour >.<
344 docstring = normalize_string_prefix(leaf.value)
345 prefix = get_string_prefix(docstring)
346 docstring = docstring[len(prefix) :] # Remove the prefix
347 quote_char = docstring[0]
348 # A natural way to remove the outer quotes is to do:
349 # docstring = docstring.strip(quote_char)
350 # but that breaks on """""x""" (which is '""x').
351 # So we actually need to remove the first character and the next two
352 # characters but only if they are the same as the first.
353 quote_len = 1 if docstring[1] != quote_char else 3
354 docstring = docstring[quote_len:-quote_len]
355 docstring_started_empty = not docstring
356 indent = " " * 4 * self.current_line.depth
358 if is_multiline_string(leaf):
359 docstring = fix_docstring(docstring, indent)
361 docstring = docstring.strip()
364 # Add some padding if the docstring starts / ends with a quote mark.
365 if docstring[0] == quote_char:
366 docstring = " " + docstring
367 if docstring[-1] == quote_char:
369 if docstring[-1] == "\\":
370 backslash_count = len(docstring) - len(docstring.rstrip("\\"))
371 if backslash_count % 2:
372 # Odd number of tailing backslashes, add some padding to
373 # avoid escaping the closing string quote.
375 elif not docstring_started_empty:
378 # We could enforce triple quotes at this point.
379 quote = quote_char * quote_len
381 # It's invalid to put closing single-character quotes on a new line.
382 if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
383 # We need to find the length of the last line of the docstring
384 # to find if we can add the closing quotes to the line without
385 # exceeding the maximum line length.
386 # If docstring is one line, then we need to add the length
387 # of the indent, prefix, and starting quotes. Ending quotes are
389 lines = docstring.splitlines()
390 last_line_length = len(lines[-1]) if docstring else 0
393 last_line_length += len(indent) + len(prefix) + quote_len
395 # If adding closing quotes would cause the last line to exceed
396 # the maximum line length then put a line break before the
398 if last_line_length + quote_len > self.mode.line_length:
399 leaf.value = prefix + quote + docstring + "\n" + indent + quote
401 leaf.value = prefix + quote + docstring + quote
403 leaf.value = prefix + quote + docstring + quote
405 yield from self.visit_default(leaf)
407 def __post_init__(self) -> None:
408 """You are in a twisty little maze of passages."""
409 self.current_line = Line(mode=self.mode)
413 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
414 self.visit_if_stmt = partial(
415 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
417 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
418 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
419 self.visit_try_stmt = partial(
420 v, keywords={"try", "except", "else", "finally"}, parens=Ø
422 if self.mode.preview:
423 self.visit_except_clause = partial(
424 v, keywords={"except"}, parens={"except"}
426 self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
428 self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
429 self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
430 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
431 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
432 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
433 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
434 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
435 self.visit_async_funcdef = self.visit_async_stmt
436 self.visit_decorated = self.visit_decorators
439 self.visit_match_stmt = self.visit_match_case
440 self.visit_case_block = self.visit_match_case
444 line: Line, mode: Mode, features: Collection[Feature] = ()
446 """Transform a `line`, potentially splitting it into many lines.
448 They should fit in the allotted `line_length` but might not be able to.
450 `features` are syntactical features that may be used in the output.
456 line_str = line_to_string(line)
458 ll = mode.line_length
459 sn = mode.string_normalization
460 string_merge = StringMerger(ll, sn)
461 string_paren_strip = StringParenStripper(ll, sn)
462 string_split = StringSplitter(ll, sn)
463 string_paren_wrap = StringParenWrapper(ll, sn)
465 transformers: List[Transformer]
467 not line.contains_uncollapsable_type_comments()
468 and not line.should_split_rhs
469 and not line.magic_trailing_comma
471 is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
472 or line.contains_unsplittable_type_ignore()
474 and not (line.inside_brackets and line.contains_standalone_comments())
476 # Only apply basic string preprocessing, since lines shouldn't be split here.
477 if Preview.string_processing in mode:
478 transformers = [string_merge, string_paren_strip]
482 transformers = [left_hand_split]
486 self: object, line: Line, features: Collection[Feature]
488 """Wraps calls to `right_hand_split`.
490 The calls increasingly `omit` right-hand trailers (bracket pairs with
491 content), meaning the trailers get glued together to split on another
492 bracket pair instead.
494 for omit in generate_trailers_to_omit(line, mode.line_length):
496 right_hand_split(line, mode.line_length, features, omit=omit)
498 # Note: this check is only able to figure out if the first line of the
499 # *current* transformation fits in the line length. This is true only
500 # for simple cases. All others require running more transforms via
501 # `transform_line()`. This check doesn't know if those would succeed.
502 if is_line_short_enough(lines[0], line_length=mode.line_length):
506 # All splits failed, best effort split with no omits.
507 # This mostly happens to multiline strings that are by definition
508 # reported as not fitting a single line, as well as lines that contain
509 # trailing commas (those have to be exploded).
510 yield from right_hand_split(
511 line, line_length=mode.line_length, features=features
514 # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
515 # __name__ attribute which is needed in `run_transformer` further down.
516 # Unfortunately a nested class breaks mypyc too. So a class must be created
517 # via type ... https://github.com/mypyc/mypyc/issues/884
518 rhs = type("rhs", (), {"__call__": _rhs})()
520 if Preview.string_processing in mode:
521 if line.inside_brackets:
527 standalone_comment_split,
540 if line.inside_brackets:
541 transformers = [delimiter_split, standalone_comment_split, rhs]
544 # It's always safe to attempt hugging of power operations and pretty much every line
546 transformers.append(hug_power_op)
548 for transform in transformers:
549 # We are accumulating lines in `result` because we might want to abort
550 # mission and return the original line in the end, or attempt a different
553 result = run_transformer(line, transform, mode, features, line_str=line_str)
554 except CannotTransform:
564 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
565 """Split line into many lines, starting with the first matching bracket pair.
567 Note: this usually looks weird, only use this for function definitions.
568 Prefer RHS otherwise. This is why this function is not symmetrical with
569 :func:`right_hand_split` which also handles optional parentheses.
571 tail_leaves: List[Leaf] = []
572 body_leaves: List[Leaf] = []
573 head_leaves: List[Leaf] = []
574 current_leaves = head_leaves
575 matching_bracket: Optional[Leaf] = None
576 for leaf in line.leaves:
578 current_leaves is body_leaves
579 and leaf.type in CLOSING_BRACKETS
580 and leaf.opening_bracket is matching_bracket
581 and isinstance(matching_bracket, Leaf)
584 ensure_visible(matching_bracket)
585 current_leaves = tail_leaves if body_leaves else head_leaves
586 current_leaves.append(leaf)
587 if current_leaves is head_leaves:
588 if leaf.type in OPENING_BRACKETS:
589 matching_bracket = leaf
590 current_leaves = body_leaves
591 if not matching_bracket:
592 raise CannotSplit("No brackets found")
594 head = bracket_split_build_line(head_leaves, line, matching_bracket)
595 body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
596 tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
597 bracket_split_succeeded_or_raise(head, body, tail)
598 for result in (head, body, tail):
603 def right_hand_split(
606 features: Collection[Feature] = (),
607 omit: Collection[LeafID] = (),
609 """Split line into many lines, starting with the last matching bracket pair.
611 If the split was by optional parentheses, attempt splitting without them, too.
612 `omit` is a collection of closing bracket IDs that shouldn't be considered for
615 Note: running this function modifies `bracket_depth` on the leaves of `line`.
617 tail_leaves: List[Leaf] = []
618 body_leaves: List[Leaf] = []
619 head_leaves: List[Leaf] = []
620 current_leaves = tail_leaves
621 opening_bracket: Optional[Leaf] = None
622 closing_bracket: Optional[Leaf] = None
623 for leaf in reversed(line.leaves):
624 if current_leaves is body_leaves:
625 if leaf is opening_bracket:
626 current_leaves = head_leaves if body_leaves else tail_leaves
627 current_leaves.append(leaf)
628 if current_leaves is tail_leaves:
629 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
630 opening_bracket = leaf.opening_bracket
631 closing_bracket = leaf
632 current_leaves = body_leaves
633 if not (opening_bracket and closing_bracket and head_leaves):
634 # If there is no opening or closing_bracket that means the split failed and
635 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
636 # the matching `opening_bracket` wasn't available on `line` anymore.
637 raise CannotSplit("No brackets found")
639 tail_leaves.reverse()
640 body_leaves.reverse()
641 head_leaves.reverse()
642 head = bracket_split_build_line(head_leaves, line, opening_bracket)
643 body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
644 tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
645 bracket_split_succeeded_or_raise(head, body, tail)
647 Feature.FORCE_OPTIONAL_PARENTHESES not in features
648 # the opening bracket is an optional paren
649 and opening_bracket.type == token.LPAR
650 and not opening_bracket.value
651 # the closing bracket is an optional paren
652 and closing_bracket.type == token.RPAR
653 and not closing_bracket.value
654 # it's not an import (optional parens are the only thing we can split on
655 # in this case; attempting a split without them is a waste of time)
656 and not line.is_import
657 # there are no standalone comments in the body
658 and not body.contains_standalone_comments(0)
659 # and we can actually remove the parens
660 and can_omit_invisible_parens(body, line_length)
662 omit = {id(closing_bracket), *omit}
664 yield from right_hand_split(line, line_length, features=features, omit=omit)
667 except CannotSplit as e:
670 or is_line_short_enough(body, line_length=line_length)
673 "Splitting failed, body is still too long and can't be split."
676 elif head.contains_multiline_strings() or tail.contains_multiline_strings():
678 "The current optional pair of parentheses is bound to fail to"
679 " satisfy the splitting algorithm because the head or the tail"
680 " contains multiline strings which by definition never fit one"
684 ensure_visible(opening_bracket)
685 ensure_visible(closing_bracket)
686 for result in (head, body, tail):
691 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
692 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
694 Do nothing otherwise.
696 A left- or right-hand split is based on a pair of brackets. Content before
697 (and including) the opening bracket is left on one line, content inside the
698 brackets is put on a separate line, and finally content starting with and
699 following the closing bracket is put on a separate line.
701 Those are called `head`, `body`, and `tail`, respectively. If the split
702 produced the same line (all content in `head`) or ended up with an empty `body`
703 and the `tail` is just the closing bracket, then it's considered failed.
705 tail_len = len(str(tail).strip())
708 raise CannotSplit("Splitting brackets produced the same line")
712 f"Splitting brackets on an empty body to save {tail_len} characters is"
717 def bracket_split_build_line(
718 leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
720 """Return a new line with given `leaves` and respective comments from `original`.
722 If `is_body` is True, the result line is one-indented inside brackets and as such
723 has its first leaf's prefix normalized and a trailing comma added when expected.
725 result = Line(mode=original.mode, depth=original.depth)
727 result.inside_brackets = True
730 # Since body is a new indent level, remove spurious leading whitespace.
731 normalize_prefix(leaves[0], inside_brackets=True)
732 # Ensure a trailing comma for imports and standalone function arguments, but
733 # be careful not to add one after any comments or within type annotations.
736 and opening_bracket.value == "("
737 and not any(leaf.type == token.COMMA for leaf in leaves)
738 # In particular, don't add one within a parenthesized return annotation.
739 # Unfortunately the indicator we're in a return annotation (RARROW) may
740 # be defined directly in the parent node, the parent of the parent ...
741 # and so on depending on how complex the return annotation is.
742 # This isn't perfect and there's some false negatives but they are in
743 # contexts were a comma is actually fine.
745 node.prev_sibling.type == RARROW
748 getattr(leaves[0].parent, "parent", None),
750 if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
754 if original.is_import or no_commas:
755 for i in range(len(leaves) - 1, -1, -1):
756 if leaves[i].type == STANDALONE_COMMENT:
759 if leaves[i].type != token.COMMA:
760 new_comma = Leaf(token.COMMA, ",")
761 leaves.insert(i + 1, new_comma)
766 result.append(leaf, preformatted=True)
767 for comment_after in original.comments_after(leaf):
768 result.append(comment_after, preformatted=True)
769 if is_body and should_split_line(result, opening_bracket):
770 result.should_split_rhs = True
774 def dont_increase_indentation(split_func: Transformer) -> Transformer:
775 """Normalize prefix of the first leaf in every line returned by `split_func`.
777 This is a decorator over relevant split functions.
781 def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
782 for split_line in split_func(line, features):
783 normalize_prefix(split_line.leaves[0], inside_brackets=True)
789 @dont_increase_indentation
790 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
791 """Split according to delimiters of the highest priority.
793 If the appropriate Features are given, the split will add trailing commas
794 also in function signatures and calls that contain `*` and `**`.
797 last_leaf = line.leaves[-1]
799 raise CannotSplit("Line empty") from None
801 bt = line.bracket_tracker
803 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
805 raise CannotSplit("No delimiters found") from None
807 if delimiter_priority == DOT_PRIORITY:
808 if bt.delimiter_count_with_priority(delimiter_priority) == 1:
809 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
812 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
814 lowest_depth = sys.maxsize
815 trailing_comma_safe = True
817 def append_to_line(leaf: Leaf) -> Iterator[Line]:
818 """Append `leaf` to current line or to new line if appending impossible."""
819 nonlocal current_line
821 current_line.append_safe(leaf, preformatted=True)
826 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
828 current_line.append(leaf)
830 for leaf in line.leaves:
831 yield from append_to_line(leaf)
833 for comment_after in line.comments_after(leaf):
834 yield from append_to_line(comment_after)
836 lowest_depth = min(lowest_depth, leaf.bracket_depth)
837 if leaf.bracket_depth == lowest_depth:
838 if is_vararg(leaf, within={syms.typedargslist}):
839 trailing_comma_safe = (
840 trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
842 elif is_vararg(leaf, within={syms.arglist, syms.argument}):
843 trailing_comma_safe = (
844 trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
847 leaf_priority = bt.delimiters.get(id(leaf))
848 if leaf_priority == delimiter_priority:
852 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
857 and delimiter_priority == COMMA_PRIORITY
858 and current_line.leaves[-1].type != token.COMMA
859 and current_line.leaves[-1].type != STANDALONE_COMMENT
861 new_comma = Leaf(token.COMMA, ",")
862 current_line.append(new_comma)
866 @dont_increase_indentation
867 def standalone_comment_split(
868 line: Line, features: Collection[Feature] = ()
870 """Split standalone comments from the rest of the line."""
871 if not line.contains_standalone_comments(0):
872 raise CannotSplit("Line does not have any standalone comments")
875 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
878 def append_to_line(leaf: Leaf) -> Iterator[Line]:
879 """Append `leaf` to current line or to new line if appending impossible."""
880 nonlocal current_line
882 current_line.append_safe(leaf, preformatted=True)
887 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
889 current_line.append(leaf)
891 for leaf in line.leaves:
892 yield from append_to_line(leaf)
894 for comment_after in line.comments_after(leaf):
895 yield from append_to_line(comment_after)
901 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
902 """Leave existing extra newlines if not `inside_brackets`. Remove everything
905 Note: don't use backslashes for formatting or you'll lose your voting rights.
907 if not inside_brackets:
908 spl = leaf.prefix.split("#")
909 if "\\" not in spl[0]:
910 nl_count = spl[-1].count("\n")
913 leaf.prefix = "\n" * nl_count
919 def normalize_invisible_parens(
920 node: Node, parens_after: Set[str], *, preview: bool
922 """Make existing optional parentheses invisible or create new ones.
924 `parens_after` is a set of string leaf values immediately after which parens
927 Standardizes on visible parentheses for single-element tuples, and keeps
928 existing visible parentheses for other tuples and generator expressions.
930 for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
931 if pc.value in FMT_OFF:
932 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
935 for index, child in enumerate(list(node.children)):
936 # Fixes a bug where invisible parens are not properly stripped from
937 # assignment statements that contain type annotations.
938 if isinstance(child, Node) and child.type == syms.annassign:
939 normalize_invisible_parens(
940 child, parens_after=parens_after, preview=preview
943 # Add parentheses around long tuple unpacking in assignments.
946 and isinstance(child, Node)
947 and child.type == syms.testlist_star_expr
954 and child.type == syms.atom
955 and node.type == syms.for_stmt
956 and isinstance(child.prev_sibling, Leaf)
957 and child.prev_sibling.type == token.NAME
958 and child.prev_sibling.value == "for"
960 if maybe_make_parens_invisible_in_atom(
963 remove_brackets_around_comma=True,
965 wrap_in_parentheses(node, child, visible=False)
966 elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
967 remove_with_parens(child, node)
968 elif child.type == syms.atom:
969 if maybe_make_parens_invisible_in_atom(
973 wrap_in_parentheses(node, child, visible=False)
974 elif is_one_tuple(child):
975 wrap_in_parentheses(node, child, visible=True)
976 elif node.type == syms.import_from:
977 # "import from" nodes store parentheses directly as part of
979 if is_lpar_token(child):
980 assert is_rpar_token(node.children[-1])
981 # make parentheses invisible
983 node.children[-1].value = ""
984 elif child.type != token.STAR:
985 # insert invisible parentheses
986 node.insert_child(index, Leaf(token.LPAR, ""))
987 node.append_child(Leaf(token.RPAR, ""))
991 and child.type == token.STAR
992 and node.type == syms.except_clause
994 # In except* (PEP 654), the star is actually part of
995 # of the keyword. So we need to skip the insertion of
996 # invisible parentheses to work more precisely.
999 elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1000 wrap_in_parentheses(node, child, visible=False)
1002 comma_check = child.type == token.COMMA if preview else False
1004 check_lpar = isinstance(child, Leaf) and (
1005 child.value in parens_after or comma_check
1009 def remove_await_parens(node: Node) -> None:
1010 if node.children[0].type == token.AWAIT and len(node.children) > 1:
1012 node.children[1].type == syms.atom
1013 and node.children[1].children[0].type == token.LPAR
1015 if maybe_make_parens_invisible_in_atom(
1018 remove_brackets_around_comma=True,
1020 wrap_in_parentheses(node, node.children[1], visible=False)
1022 # Since await is an expression we shouldn't remove
1023 # brackets in cases where this would change
1024 # the AST due to operator precedence.
1025 # Therefore we only aim to remove brackets around
1026 # power nodes that aren't also await expressions themselves.
1027 # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1028 # N.B. We've still removed any redundant nested brackets though :)
1029 opening_bracket = cast(Leaf, node.children[1].children[0])
1030 closing_bracket = cast(Leaf, node.children[1].children[-1])
1031 bracket_contents = cast(Node, node.children[1].children[1])
1032 if bracket_contents.type != syms.power:
1033 ensure_visible(opening_bracket)
1034 ensure_visible(closing_bracket)
1036 bracket_contents.type == syms.power
1037 and bracket_contents.children[0].type == token.AWAIT
1039 ensure_visible(opening_bracket)
1040 ensure_visible(closing_bracket)
1041 # If we are in a nested await then recurse down.
1042 remove_await_parens(bracket_contents)
1045 def remove_with_parens(node: Node, parent: Node) -> None:
1046 """Recursively hide optional parens in `with` statements."""
1047 # Removing all unnecessary parentheses in with statements in one pass is a tad
1048 # complex as different variations of bracketed statements result in pretty
1049 # different parse trees:
1051 # with (open("file")) as f: # this is an asexpr_test
1054 # with (open("file") as f): # this is an atom containing an
1057 # with (open("file")) as f, (open("file")) as f: # this is asexpr_test, COMMA,
1060 # with (open("file") as f, open("file") as f): # an atom containing a
1061 # ... # testlist_gexp which then
1062 # # contains multiple asexpr_test(s)
1063 if node.type == syms.atom:
1064 if maybe_make_parens_invisible_in_atom(
1067 remove_brackets_around_comma=True,
1069 wrap_in_parentheses(parent, node, visible=False)
1070 if isinstance(node.children[1], Node):
1071 remove_with_parens(node.children[1], node)
1072 elif node.type == syms.testlist_gexp:
1073 for child in node.children:
1074 if isinstance(child, Node):
1075 remove_with_parens(child, node)
1076 elif node.type == syms.asexpr_test and not any(
1077 leaf.type == token.COLONEQUAL for leaf in node.leaves()
1079 if maybe_make_parens_invisible_in_atom(
1082 remove_brackets_around_comma=True,
1084 wrap_in_parentheses(node, node.children[0], visible=False)
1087 def maybe_make_parens_invisible_in_atom(
1090 remove_brackets_around_comma: bool = False,
1092 """If it's safe, make the parens in the atom `node` invisible, recursively.
1093 Additionally, remove repeated, adjacent invisible parens from the atom `node`
1094 as they are redundant.
1096 Returns whether the node should itself be wrapped in invisible parentheses.
1099 node.type != syms.atom
1100 or is_empty_tuple(node)
1101 or is_one_tuple(node)
1102 or (is_yield(node) and parent.type != syms.expr_stmt)
1104 # This condition tries to prevent removing non-optional brackets
1105 # around a tuple, however, can be a bit overzealous so we provide
1106 # and option to skip this check for `for` and `with` statements.
1107 not remove_brackets_around_comma
1108 and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1113 if is_walrus_assignment(node):
1119 # these ones aren't useful to end users, but they do please fuzzers
1125 first = node.children[0]
1126 last = node.children[-1]
1127 if is_lpar_token(first) and is_rpar_token(last):
1128 middle = node.children[1]
1129 # make parentheses invisible
1132 maybe_make_parens_invisible_in_atom(
1135 remove_brackets_around_comma=remove_brackets_around_comma,
1138 if is_atom_with_invisible_parens(middle):
1139 # Strip the invisible parens from `middle` by replacing
1140 # it with the child in-between the invisible parens
1141 middle.replace(middle.children[1])
1148 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1149 """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1151 if not (opening_bracket.parent and opening_bracket.value in "[{("):
1154 # We're essentially checking if the body is delimited by commas and there's more
1155 # than one of them (we're excluding the trailing comma and if the delimiter priority
1156 # is still commas, that means there's more).
1158 trailing_comma = False
1160 last_leaf = line.leaves[-1]
1161 if last_leaf.type == token.COMMA:
1162 trailing_comma = True
1163 exclude.add(id(last_leaf))
1164 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1165 except (IndexError, ValueError):
1168 return max_priority == COMMA_PRIORITY and (
1169 (line.mode.magic_trailing_comma and trailing_comma)
1170 # always explode imports
1171 or opening_bracket.parent.type in {syms.atom, syms.import_from}
1175 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1176 """Generate sets of closing bracket IDs that should be omitted in a RHS.
1178 Brackets can be omitted if the entire trailer up to and including
1179 a preceding closing bracket fits in one line.
1181 Yielded sets are cumulative (contain results of previous yields, too). First
1182 set is empty, unless the line should explode, in which case bracket pairs until
1183 the one that needs to explode are omitted.
1186 omit: Set[LeafID] = set()
1187 if not line.magic_trailing_comma:
1190 length = 4 * line.depth
1191 opening_bracket: Optional[Leaf] = None
1192 closing_bracket: Optional[Leaf] = None
1193 inner_brackets: Set[LeafID] = set()
1194 for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1195 length += leaf_length
1196 if length > line_length:
1199 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1200 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1204 if leaf is opening_bracket:
1205 opening_bracket = None
1206 elif leaf.type in CLOSING_BRACKETS:
1207 prev = line.leaves[index - 1] if index > 0 else None
1210 and prev.type == token.COMMA
1211 and leaf.opening_bracket is not None
1212 and not is_one_sequence_between(
1213 leaf.opening_bracket, leaf, line.leaves
1216 # Never omit bracket pairs with trailing commas.
1217 # We need to explode on those.
1220 inner_brackets.add(id(leaf))
1221 elif leaf.type in CLOSING_BRACKETS:
1222 prev = line.leaves[index - 1] if index > 0 else None
1223 if prev and prev.type in OPENING_BRACKETS:
1224 # Empty brackets would fail a split so treat them as "inner"
1225 # brackets (e.g. only add them to the `omit` set if another
1226 # pair of brackets was good enough.
1227 inner_brackets.add(id(leaf))
1231 omit.add(id(closing_bracket))
1232 omit.update(inner_brackets)
1233 inner_brackets.clear()
1238 and prev.type == token.COMMA
1239 and leaf.opening_bracket is not None
1240 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1242 # Never omit bracket pairs with trailing commas.
1243 # We need to explode on those.
1247 opening_bracket = leaf.opening_bracket
1248 closing_bracket = leaf
1251 def run_transformer(
1253 transform: Transformer,
1255 features: Collection[Feature],
1260 line_str = line_to_string(line)
1261 result: List[Line] = []
1262 for transformed_line in transform(line, features):
1263 if str(transformed_line).strip("\n") == line_str:
1264 raise CannotTransform("Line transformer returned an unchanged result")
1266 result.extend(transform_line(transformed_line, mode=mode, features=features))
1269 transform.__class__.__name__ != "rhs"
1270 or not line.bracket_tracker.invisible
1271 or any(bracket.value for bracket in line.bracket_tracker.invisible)
1272 or line.contains_multiline_strings()
1273 or result[0].contains_uncollapsable_type_comments()
1274 or result[0].contains_unsplittable_type_ignore()
1275 or is_line_short_enough(result[0], line_length=mode.line_length)
1276 # If any leaves have no parents (which _can_ occur since
1277 # `transform(line)` potentially destroys the line's underlying node
1278 # structure), then we can't proceed. Doing so would cause the below
1279 # call to `append_leaves()` to fail.
1280 or any(leaf.parent is None for leaf in line.leaves)
1284 line_copy = line.clone()
1285 append_leaves(line_copy, line, line.leaves)
1286 features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1287 second_opinion = run_transformer(
1288 line_copy, transform, mode, features_fop, line_str=line_str
1291 is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1293 result = second_opinion