]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

4cba4164fb357f73b971ff0070fc0fd624323bc2
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 from functools import partial, wraps
5 import sys
6 from typing import Collection, Iterator, List, Optional, Set, Union
7
8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
10 from black.nodes import Visitor, syms, first_child_is_arith, ensure_visible
11 from black.nodes import is_docstring, is_empty_tuple, is_one_tuple, is_one_tuple_between
12 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
13 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
14 from black.nodes import wrap_in_parentheses
15 from black.brackets import max_delimiter_priority_in_atom
16 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
17 from black.lines import Line, line_to_string, is_line_short_enough
18 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
19 from black.comments import generate_comments, list_comments, FMT_OFF
20 from black.numerics import normalize_numeric_literal
21 from black.strings import get_string_prefix, fix_docstring
22 from black.strings import normalize_string_prefix, normalize_string_quotes
23 from black.trans import Transformer, CannotTransform, StringMerger
24 from black.trans import StringSplitter, StringParenWrapper, StringParenStripper
25 from black.mode import Mode
26 from black.mode import Feature
27
28 from blib2to3.pytree import Node, Leaf
29 from blib2to3.pgen2 import token
30
31
32 # types
33 LeafID = int
34 LN = Union[Leaf, Node]
35
36
37 class CannotSplit(CannotTransform):
38     """A readable split that fits the allotted line length is impossible."""
39
40
41 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
42 # See also https://github.com/mypyc/mypyc/issues/827.
43 class LineGenerator(Visitor[Line]):
44     """Generates reformatted Line objects.  Empty lines are not emitted.
45
46     Note: destroys the tree it's visiting by mutating prefixes of its leaves
47     in ways that will no longer stringify to valid Python code on the tree.
48     """
49
50     def __init__(self, mode: Mode, remove_u_prefix: bool = False) -> None:
51         self.mode = mode
52         self.remove_u_prefix = remove_u_prefix
53         self.current_line: Line
54         self.__post_init__()
55
56     def line(self, indent: int = 0) -> Iterator[Line]:
57         """Generate a line.
58
59         If the line is empty, only emit if it makes sense.
60         If the line is too long, split it first and then generate.
61
62         If any lines were generated, set up a new current_line.
63         """
64         if not self.current_line:
65             self.current_line.depth += indent
66             return  # Line is empty, don't emit. Creating a new one unnecessary.
67
68         complete_line = self.current_line
69         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
70         yield complete_line
71
72     def visit_default(self, node: LN) -> Iterator[Line]:
73         """Default `visit_*()` implementation. Recurses to children of `node`."""
74         if isinstance(node, Leaf):
75             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
76             for comment in generate_comments(node):
77                 if any_open_brackets:
78                     # any comment within brackets is subject to splitting
79                     self.current_line.append(comment)
80                 elif comment.type == token.COMMENT:
81                     # regular trailing comment
82                     self.current_line.append(comment)
83                     yield from self.line()
84
85                 else:
86                     # regular standalone comment
87                     yield from self.line()
88
89                     self.current_line.append(comment)
90                     yield from self.line()
91
92             normalize_prefix(node, inside_brackets=any_open_brackets)
93             if self.mode.string_normalization and node.type == token.STRING:
94                 node.value = normalize_string_prefix(
95                     node.value, remove_u_prefix=self.remove_u_prefix
96                 )
97                 node.value = normalize_string_quotes(node.value)
98             if node.type == token.NUMBER:
99                 normalize_numeric_literal(node)
100             if node.type not in WHITESPACE:
101                 self.current_line.append(node)
102         yield from super().visit_default(node)
103
104     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
105         """Increase indentation level, maybe yield a line."""
106         # In blib2to3 INDENT never holds comments.
107         yield from self.line(+1)
108         yield from self.visit_default(node)
109
110     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
111         """Decrease indentation level, maybe yield a line."""
112         # The current line might still wait for trailing comments.  At DEDENT time
113         # there won't be any (they would be prefixes on the preceding NEWLINE).
114         # Emit the line then.
115         yield from self.line()
116
117         # While DEDENT has no value, its prefix may contain standalone comments
118         # that belong to the current indentation level.  Get 'em.
119         yield from self.visit_default(node)
120
121         # Finally, emit the dedent.
122         yield from self.line(-1)
123
124     def visit_stmt(
125         self, node: Node, keywords: Set[str], parens: Set[str]
126     ) -> Iterator[Line]:
127         """Visit a statement.
128
129         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
130         `def`, `with`, `class`, `assert`, `match`, `case` and assignments.
131
132         The relevant Python language `keywords` for a given statement will be
133         NAME leaves within it. This methods puts those on a separate line.
134
135         `parens` holds a set of string leaf values immediately after which
136         invisible parens should be put.
137         """
138         normalize_invisible_parens(node, parens_after=parens)
139         for child in node.children:
140             if child.type == token.NAME and child.value in keywords:  # type: ignore
141                 yield from self.line()
142
143             yield from self.visit(child)
144
145     def visit_suite(self, node: Node) -> Iterator[Line]:
146         """Visit a suite."""
147         if self.mode.is_pyi and is_stub_suite(node):
148             yield from self.visit(node.children[2])
149         else:
150             yield from self.visit_default(node)
151
152     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
153         """Visit a statement without nested statements."""
154         if first_child_is_arith(node):
155             wrap_in_parentheses(node, node.children[0], visible=False)
156         is_suite_like = node.parent and node.parent.type in STATEMENT
157         if is_suite_like:
158             if self.mode.is_pyi and is_stub_body(node):
159                 yield from self.visit_default(node)
160             else:
161                 yield from self.line(+1)
162                 yield from self.visit_default(node)
163                 yield from self.line(-1)
164
165         else:
166             if (
167                 not self.mode.is_pyi
168                 or not node.parent
169                 or not is_stub_suite(node.parent)
170             ):
171                 yield from self.line()
172             yield from self.visit_default(node)
173
174     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
175         """Visit `async def`, `async for`, `async with`."""
176         yield from self.line()
177
178         children = iter(node.children)
179         for child in children:
180             yield from self.visit(child)
181
182             if child.type == token.ASYNC:
183                 break
184
185         internal_stmt = next(children)
186         for child in internal_stmt.children:
187             yield from self.visit(child)
188
189     def visit_decorators(self, node: Node) -> Iterator[Line]:
190         """Visit decorators."""
191         for child in node.children:
192             yield from self.line()
193             yield from self.visit(child)
194
195     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
196         """Remove a semicolon and put the other statement on a separate line."""
197         yield from self.line()
198
199     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
200         """End of file. Process outstanding comments and end with a newline."""
201         yield from self.visit_default(leaf)
202         yield from self.line()
203
204     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
205         if not self.current_line.bracket_tracker.any_open_brackets():
206             yield from self.line()
207         yield from self.visit_default(leaf)
208
209     def visit_factor(self, node: Node) -> Iterator[Line]:
210         """Force parentheses between a unary op and a binary power:
211
212         -2 ** 8 -> -(2 ** 8)
213         """
214         _operator, operand = node.children
215         if (
216             operand.type == syms.power
217             and len(operand.children) == 3
218             and operand.children[1].type == token.DOUBLESTAR
219         ):
220             lpar = Leaf(token.LPAR, "(")
221             rpar = Leaf(token.RPAR, ")")
222             index = operand.remove() or 0
223             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
224         yield from self.visit_default(node)
225
226     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
227         if is_docstring(leaf) and "\\\n" not in leaf.value:
228             # We're ignoring docstrings with backslash newline escapes because changing
229             # indentation of those changes the AST representation of the code.
230             docstring = normalize_string_prefix(leaf.value, self.remove_u_prefix)
231             prefix = get_string_prefix(docstring)
232             docstring = docstring[len(prefix) :]  # Remove the prefix
233             quote_char = docstring[0]
234             # A natural way to remove the outer quotes is to do:
235             #   docstring = docstring.strip(quote_char)
236             # but that breaks on """""x""" (which is '""x').
237             # So we actually need to remove the first character and the next two
238             # characters but only if they are the same as the first.
239             quote_len = 1 if docstring[1] != quote_char else 3
240             docstring = docstring[quote_len:-quote_len]
241             docstring_started_empty = not docstring
242
243             if is_multiline_string(leaf):
244                 indent = " " * 4 * self.current_line.depth
245                 docstring = fix_docstring(docstring, indent)
246             else:
247                 docstring = docstring.strip()
248
249             if docstring:
250                 # Add some padding if the docstring starts / ends with a quote mark.
251                 if docstring[0] == quote_char:
252                     docstring = " " + docstring
253                 if docstring[-1] == quote_char:
254                     docstring += " "
255                 if docstring[-1] == "\\":
256                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
257                     if backslash_count % 2:
258                         # Odd number of tailing backslashes, add some padding to
259                         # avoid escaping the closing string quote.
260                         docstring += " "
261             elif not docstring_started_empty:
262                 docstring = " "
263
264             # We could enforce triple quotes at this point.
265             quote = quote_char * quote_len
266             leaf.value = prefix + quote + docstring + quote
267
268         yield from self.visit_default(leaf)
269
270     def __post_init__(self) -> None:
271         """You are in a twisty little maze of passages."""
272         self.current_line = Line(mode=self.mode)
273
274         v = self.visit_stmt
275         Ø: Set[str] = set()
276         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
277         self.visit_if_stmt = partial(
278             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
279         )
280         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
281         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
282         self.visit_try_stmt = partial(
283             v, keywords={"try", "except", "else", "finally"}, parens=Ø
284         )
285         self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
286         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
287         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
288         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
289         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
290         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
291         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
292         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
293         self.visit_async_funcdef = self.visit_async_stmt
294         self.visit_decorated = self.visit_decorators
295
296         # PEP 634
297         self.visit_match_stmt = partial(v, keywords={"match"}, parens=Ø)
298         self.visit_case_block = partial(v, keywords={"case"}, parens=Ø)
299
300
301 def transform_line(
302     line: Line, mode: Mode, features: Collection[Feature] = ()
303 ) -> Iterator[Line]:
304     """Transform a `line`, potentially splitting it into many lines.
305
306     They should fit in the allotted `line_length` but might not be able to.
307
308     `features` are syntactical features that may be used in the output.
309     """
310     if line.is_comment:
311         yield line
312         return
313
314     line_str = line_to_string(line)
315
316     ll = mode.line_length
317     sn = mode.string_normalization
318     string_merge = StringMerger(ll, sn)
319     string_paren_strip = StringParenStripper(ll, sn)
320     string_split = StringSplitter(ll, sn)
321     string_paren_wrap = StringParenWrapper(ll, sn)
322
323     transformers: List[Transformer]
324     if (
325         not line.contains_uncollapsable_type_comments()
326         and not line.should_split_rhs
327         and not line.magic_trailing_comma
328         and (
329             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
330             or line.contains_unsplittable_type_ignore()
331         )
332         and not (line.inside_brackets and line.contains_standalone_comments())
333     ):
334         # Only apply basic string preprocessing, since lines shouldn't be split here.
335         if mode.experimental_string_processing:
336             transformers = [string_merge, string_paren_strip]
337         else:
338             transformers = []
339     elif line.is_def:
340         transformers = [left_hand_split]
341     else:
342
343         def _rhs(
344             self: object, line: Line, features: Collection[Feature]
345         ) -> Iterator[Line]:
346             """Wraps calls to `right_hand_split`.
347
348             The calls increasingly `omit` right-hand trailers (bracket pairs with
349             content), meaning the trailers get glued together to split on another
350             bracket pair instead.
351             """
352             for omit in generate_trailers_to_omit(line, mode.line_length):
353                 lines = list(
354                     right_hand_split(line, mode.line_length, features, omit=omit)
355                 )
356                 # Note: this check is only able to figure out if the first line of the
357                 # *current* transformation fits in the line length.  This is true only
358                 # for simple cases.  All others require running more transforms via
359                 # `transform_line()`.  This check doesn't know if those would succeed.
360                 if is_line_short_enough(lines[0], line_length=mode.line_length):
361                     yield from lines
362                     return
363
364             # All splits failed, best effort split with no omits.
365             # This mostly happens to multiline strings that are by definition
366             # reported as not fitting a single line, as well as lines that contain
367             # trailing commas (those have to be exploded).
368             yield from right_hand_split(
369                 line, line_length=mode.line_length, features=features
370             )
371
372         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
373         # __name__ attribute which is needed in `run_transformer` further down.
374         # Unfortunately a nested class breaks mypyc too. So a class must be created
375         # via type ... https://github.com/mypyc/mypyc/issues/884
376         rhs = type("rhs", (), {"__call__": _rhs})()
377
378         if mode.experimental_string_processing:
379             if line.inside_brackets:
380                 transformers = [
381                     string_merge,
382                     string_paren_strip,
383                     string_split,
384                     delimiter_split,
385                     standalone_comment_split,
386                     string_paren_wrap,
387                     rhs,
388                 ]
389             else:
390                 transformers = [
391                     string_merge,
392                     string_paren_strip,
393                     string_split,
394                     string_paren_wrap,
395                     rhs,
396                 ]
397         else:
398             if line.inside_brackets:
399                 transformers = [delimiter_split, standalone_comment_split, rhs]
400             else:
401                 transformers = [rhs]
402
403     for transform in transformers:
404         # We are accumulating lines in `result` because we might want to abort
405         # mission and return the original line in the end, or attempt a different
406         # split altogether.
407         try:
408             result = run_transformer(line, transform, mode, features, line_str=line_str)
409         except CannotTransform:
410             continue
411         else:
412             yield from result
413             break
414
415     else:
416         yield line
417
418
419 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
420     """Split line into many lines, starting with the first matching bracket pair.
421
422     Note: this usually looks weird, only use this for function definitions.
423     Prefer RHS otherwise.  This is why this function is not symmetrical with
424     :func:`right_hand_split` which also handles optional parentheses.
425     """
426     tail_leaves: List[Leaf] = []
427     body_leaves: List[Leaf] = []
428     head_leaves: List[Leaf] = []
429     current_leaves = head_leaves
430     matching_bracket: Optional[Leaf] = None
431     for leaf in line.leaves:
432         if (
433             current_leaves is body_leaves
434             and leaf.type in CLOSING_BRACKETS
435             and leaf.opening_bracket is matching_bracket
436         ):
437             current_leaves = tail_leaves if body_leaves else head_leaves
438         current_leaves.append(leaf)
439         if current_leaves is head_leaves:
440             if leaf.type in OPENING_BRACKETS:
441                 matching_bracket = leaf
442                 current_leaves = body_leaves
443     if not matching_bracket:
444         raise CannotSplit("No brackets found")
445
446     head = bracket_split_build_line(head_leaves, line, matching_bracket)
447     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
448     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
449     bracket_split_succeeded_or_raise(head, body, tail)
450     for result in (head, body, tail):
451         if result:
452             yield result
453
454
455 def right_hand_split(
456     line: Line,
457     line_length: int,
458     features: Collection[Feature] = (),
459     omit: Collection[LeafID] = (),
460 ) -> Iterator[Line]:
461     """Split line into many lines, starting with the last matching bracket pair.
462
463     If the split was by optional parentheses, attempt splitting without them, too.
464     `omit` is a collection of closing bracket IDs that shouldn't be considered for
465     this split.
466
467     Note: running this function modifies `bracket_depth` on the leaves of `line`.
468     """
469     tail_leaves: List[Leaf] = []
470     body_leaves: List[Leaf] = []
471     head_leaves: List[Leaf] = []
472     current_leaves = tail_leaves
473     opening_bracket: Optional[Leaf] = None
474     closing_bracket: Optional[Leaf] = None
475     for leaf in reversed(line.leaves):
476         if current_leaves is body_leaves:
477             if leaf is opening_bracket:
478                 current_leaves = head_leaves if body_leaves else tail_leaves
479         current_leaves.append(leaf)
480         if current_leaves is tail_leaves:
481             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
482                 opening_bracket = leaf.opening_bracket
483                 closing_bracket = leaf
484                 current_leaves = body_leaves
485     if not (opening_bracket and closing_bracket and head_leaves):
486         # If there is no opening or closing_bracket that means the split failed and
487         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
488         # the matching `opening_bracket` wasn't available on `line` anymore.
489         raise CannotSplit("No brackets found")
490
491     tail_leaves.reverse()
492     body_leaves.reverse()
493     head_leaves.reverse()
494     head = bracket_split_build_line(head_leaves, line, opening_bracket)
495     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
496     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
497     bracket_split_succeeded_or_raise(head, body, tail)
498     if (
499         Feature.FORCE_OPTIONAL_PARENTHESES not in features
500         # the opening bracket is an optional paren
501         and opening_bracket.type == token.LPAR
502         and not opening_bracket.value
503         # the closing bracket is an optional paren
504         and closing_bracket.type == token.RPAR
505         and not closing_bracket.value
506         # it's not an import (optional parens are the only thing we can split on
507         # in this case; attempting a split without them is a waste of time)
508         and not line.is_import
509         # there are no standalone comments in the body
510         and not body.contains_standalone_comments(0)
511         # and we can actually remove the parens
512         and can_omit_invisible_parens(body, line_length, omit_on_explode=omit)
513     ):
514         omit = {id(closing_bracket), *omit}
515         try:
516             yield from right_hand_split(line, line_length, features=features, omit=omit)
517             return
518
519         except CannotSplit as e:
520             if not (
521                 can_be_split(body)
522                 or is_line_short_enough(body, line_length=line_length)
523             ):
524                 raise CannotSplit(
525                     "Splitting failed, body is still too long and can't be split."
526                 ) from e
527
528             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
529                 raise CannotSplit(
530                     "The current optional pair of parentheses is bound to fail to"
531                     " satisfy the splitting algorithm because the head or the tail"
532                     " contains multiline strings which by definition never fit one"
533                     " line."
534                 ) from e
535
536     ensure_visible(opening_bracket)
537     ensure_visible(closing_bracket)
538     for result in (head, body, tail):
539         if result:
540             yield result
541
542
543 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
544     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
545
546     Do nothing otherwise.
547
548     A left- or right-hand split is based on a pair of brackets. Content before
549     (and including) the opening bracket is left on one line, content inside the
550     brackets is put on a separate line, and finally content starting with and
551     following the closing bracket is put on a separate line.
552
553     Those are called `head`, `body`, and `tail`, respectively. If the split
554     produced the same line (all content in `head`) or ended up with an empty `body`
555     and the `tail` is just the closing bracket, then it's considered failed.
556     """
557     tail_len = len(str(tail).strip())
558     if not body:
559         if tail_len == 0:
560             raise CannotSplit("Splitting brackets produced the same line")
561
562         elif tail_len < 3:
563             raise CannotSplit(
564                 f"Splitting brackets on an empty body to save {tail_len} characters is"
565                 " not worth it"
566             )
567
568
569 def bracket_split_build_line(
570     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
571 ) -> Line:
572     """Return a new line with given `leaves` and respective comments from `original`.
573
574     If `is_body` is True, the result line is one-indented inside brackets and as such
575     has its first leaf's prefix normalized and a trailing comma added when expected.
576     """
577     result = Line(mode=original.mode, depth=original.depth)
578     if is_body:
579         result.inside_brackets = True
580         result.depth += 1
581         if leaves:
582             # Since body is a new indent level, remove spurious leading whitespace.
583             normalize_prefix(leaves[0], inside_brackets=True)
584             # Ensure a trailing comma for imports and standalone function arguments, but
585             # be careful not to add one after any comments or within type annotations.
586             no_commas = (
587                 original.is_def
588                 and opening_bracket.value == "("
589                 and not any(leaf.type == token.COMMA for leaf in leaves)
590                 # In particular, don't add one within a parenthesized return annotation.
591                 # Unfortunately the indicator we're in a return annotation (RARROW) may
592                 # be defined directly in the parent node, the parent of the parent ...
593                 # and so on depending on how complex the return annotation is.
594                 # This isn't perfect and there's some false negatives but they are in
595                 # contexts were a comma is actually fine.
596                 and not any(
597                     node.prev_sibling.type == RARROW
598                     for node in (
599                         leaves[0].parent,
600                         getattr(leaves[0].parent, "parent", None),
601                     )
602                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
603                 )
604             )
605
606             if original.is_import or no_commas:
607                 for i in range(len(leaves) - 1, -1, -1):
608                     if leaves[i].type == STANDALONE_COMMENT:
609                         continue
610
611                     if leaves[i].type != token.COMMA:
612                         new_comma = Leaf(token.COMMA, ",")
613                         leaves.insert(i + 1, new_comma)
614                     break
615
616     # Populate the line
617     for leaf in leaves:
618         result.append(leaf, preformatted=True)
619         for comment_after in original.comments_after(leaf):
620             result.append(comment_after, preformatted=True)
621     if is_body and should_split_line(result, opening_bracket):
622         result.should_split_rhs = True
623     return result
624
625
626 def dont_increase_indentation(split_func: Transformer) -> Transformer:
627     """Normalize prefix of the first leaf in every line returned by `split_func`.
628
629     This is a decorator over relevant split functions.
630     """
631
632     @wraps(split_func)
633     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
634         for line in split_func(line, features):
635             normalize_prefix(line.leaves[0], inside_brackets=True)
636             yield line
637
638     return split_wrapper
639
640
641 @dont_increase_indentation
642 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
643     """Split according to delimiters of the highest priority.
644
645     If the appropriate Features are given, the split will add trailing commas
646     also in function signatures and calls that contain `*` and `**`.
647     """
648     try:
649         last_leaf = line.leaves[-1]
650     except IndexError:
651         raise CannotSplit("Line empty") from None
652
653     bt = line.bracket_tracker
654     try:
655         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
656     except ValueError:
657         raise CannotSplit("No delimiters found") from None
658
659     if delimiter_priority == DOT_PRIORITY:
660         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
661             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
662
663     current_line = Line(
664         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
665     )
666     lowest_depth = sys.maxsize
667     trailing_comma_safe = True
668
669     def append_to_line(leaf: Leaf) -> Iterator[Line]:
670         """Append `leaf` to current line or to new line if appending impossible."""
671         nonlocal current_line
672         try:
673             current_line.append_safe(leaf, preformatted=True)
674         except ValueError:
675             yield current_line
676
677             current_line = Line(
678                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
679             )
680             current_line.append(leaf)
681
682     for leaf in line.leaves:
683         yield from append_to_line(leaf)
684
685         for comment_after in line.comments_after(leaf):
686             yield from append_to_line(comment_after)
687
688         lowest_depth = min(lowest_depth, leaf.bracket_depth)
689         if leaf.bracket_depth == lowest_depth:
690             if is_vararg(leaf, within={syms.typedargslist}):
691                 trailing_comma_safe = (
692                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
693                 )
694             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
695                 trailing_comma_safe = (
696                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
697                 )
698
699         leaf_priority = bt.delimiters.get(id(leaf))
700         if leaf_priority == delimiter_priority:
701             yield current_line
702
703             current_line = Line(
704                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
705             )
706     if current_line:
707         if (
708             trailing_comma_safe
709             and delimiter_priority == COMMA_PRIORITY
710             and current_line.leaves[-1].type != token.COMMA
711             and current_line.leaves[-1].type != STANDALONE_COMMENT
712         ):
713             new_comma = Leaf(token.COMMA, ",")
714             current_line.append(new_comma)
715         yield current_line
716
717
718 @dont_increase_indentation
719 def standalone_comment_split(
720     line: Line, features: Collection[Feature] = ()
721 ) -> Iterator[Line]:
722     """Split standalone comments from the rest of the line."""
723     if not line.contains_standalone_comments(0):
724         raise CannotSplit("Line does not have any standalone comments")
725
726     current_line = Line(
727         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
728     )
729
730     def append_to_line(leaf: Leaf) -> Iterator[Line]:
731         """Append `leaf` to current line or to new line if appending impossible."""
732         nonlocal current_line
733         try:
734             current_line.append_safe(leaf, preformatted=True)
735         except ValueError:
736             yield current_line
737
738             current_line = Line(
739                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
740             )
741             current_line.append(leaf)
742
743     for leaf in line.leaves:
744         yield from append_to_line(leaf)
745
746         for comment_after in line.comments_after(leaf):
747             yield from append_to_line(comment_after)
748
749     if current_line:
750         yield current_line
751
752
753 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
754     """Leave existing extra newlines if not `inside_brackets`. Remove everything
755     else.
756
757     Note: don't use backslashes for formatting or you'll lose your voting rights.
758     """
759     if not inside_brackets:
760         spl = leaf.prefix.split("#")
761         if "\\" not in spl[0]:
762             nl_count = spl[-1].count("\n")
763             if len(spl) > 1:
764                 nl_count -= 1
765             leaf.prefix = "\n" * nl_count
766             return
767
768     leaf.prefix = ""
769
770
771 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
772     """Make existing optional parentheses invisible or create new ones.
773
774     `parens_after` is a set of string leaf values immediately after which parens
775     should be put.
776
777     Standardizes on visible parentheses for single-element tuples, and keeps
778     existing visible parentheses for other tuples and generator expressions.
779     """
780     for pc in list_comments(node.prefix, is_endmarker=False):
781         if pc.value in FMT_OFF:
782             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
783             return
784     check_lpar = False
785     for index, child in enumerate(list(node.children)):
786         # Fixes a bug where invisible parens are not properly stripped from
787         # assignment statements that contain type annotations.
788         if isinstance(child, Node) and child.type == syms.annassign:
789             normalize_invisible_parens(child, parens_after=parens_after)
790
791         # Add parentheses around long tuple unpacking in assignments.
792         if (
793             index == 0
794             and isinstance(child, Node)
795             and child.type == syms.testlist_star_expr
796         ):
797             check_lpar = True
798
799         if check_lpar:
800             if child.type == syms.atom:
801                 if maybe_make_parens_invisible_in_atom(child, parent=node):
802                     wrap_in_parentheses(node, child, visible=False)
803             elif is_one_tuple(child):
804                 wrap_in_parentheses(node, child, visible=True)
805             elif node.type == syms.import_from:
806                 # "import from" nodes store parentheses directly as part of
807                 # the statement
808                 if child.type == token.LPAR:
809                     # make parentheses invisible
810                     child.value = ""  # type: ignore
811                     node.children[-1].value = ""  # type: ignore
812                 elif child.type != token.STAR:
813                     # insert invisible parentheses
814                     node.insert_child(index, Leaf(token.LPAR, ""))
815                     node.append_child(Leaf(token.RPAR, ""))
816                 break
817
818             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
819                 wrap_in_parentheses(node, child, visible=False)
820
821         check_lpar = isinstance(child, Leaf) and child.value in parens_after
822
823
824 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
825     """If it's safe, make the parens in the atom `node` invisible, recursively.
826     Additionally, remove repeated, adjacent invisible parens from the atom `node`
827     as they are redundant.
828
829     Returns whether the node should itself be wrapped in invisible parentheses.
830
831     """
832
833     if (
834         node.type != syms.atom
835         or is_empty_tuple(node)
836         or is_one_tuple(node)
837         or (is_yield(node) and parent.type != syms.expr_stmt)
838         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
839     ):
840         return False
841
842     if is_walrus_assignment(node):
843         if parent.type in [
844             syms.annassign,
845             syms.expr_stmt,
846             syms.assert_stmt,
847             syms.return_stmt,
848             # these ones aren't useful to end users, but they do please fuzzers
849             syms.for_stmt,
850             syms.del_stmt,
851         ]:
852             return False
853
854     first = node.children[0]
855     last = node.children[-1]
856     if first.type == token.LPAR and last.type == token.RPAR:
857         middle = node.children[1]
858         # make parentheses invisible
859         first.value = ""  # type: ignore
860         last.value = ""  # type: ignore
861         maybe_make_parens_invisible_in_atom(middle, parent=parent)
862
863         if is_atom_with_invisible_parens(middle):
864             # Strip the invisible parens from `middle` by replacing
865             # it with the child in-between the invisible parens
866             middle.replace(middle.children[1])
867
868         return False
869
870     return True
871
872
873 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
874     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
875
876     if not (opening_bracket.parent and opening_bracket.value in "[{("):
877         return False
878
879     # We're essentially checking if the body is delimited by commas and there's more
880     # than one of them (we're excluding the trailing comma and if the delimiter priority
881     # is still commas, that means there's more).
882     exclude = set()
883     trailing_comma = False
884     try:
885         last_leaf = line.leaves[-1]
886         if last_leaf.type == token.COMMA:
887             trailing_comma = True
888             exclude.add(id(last_leaf))
889         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
890     except (IndexError, ValueError):
891         return False
892
893     return max_priority == COMMA_PRIORITY and (
894         (line.mode.magic_trailing_comma and trailing_comma)
895         # always explode imports
896         or opening_bracket.parent.type in {syms.atom, syms.import_from}
897     )
898
899
900 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
901     """Generate sets of closing bracket IDs that should be omitted in a RHS.
902
903     Brackets can be omitted if the entire trailer up to and including
904     a preceding closing bracket fits in one line.
905
906     Yielded sets are cumulative (contain results of previous yields, too).  First
907     set is empty, unless the line should explode, in which case bracket pairs until
908     the one that needs to explode are omitted.
909     """
910
911     omit: Set[LeafID] = set()
912     if not line.magic_trailing_comma:
913         yield omit
914
915     length = 4 * line.depth
916     opening_bracket: Optional[Leaf] = None
917     closing_bracket: Optional[Leaf] = None
918     inner_brackets: Set[LeafID] = set()
919     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
920         length += leaf_length
921         if length > line_length:
922             break
923
924         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
925         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
926             break
927
928         if opening_bracket:
929             if leaf is opening_bracket:
930                 opening_bracket = None
931             elif leaf.type in CLOSING_BRACKETS:
932                 prev = line.leaves[index - 1] if index > 0 else None
933                 if (
934                     prev
935                     and prev.type == token.COMMA
936                     and not is_one_tuple_between(
937                         leaf.opening_bracket, leaf, line.leaves
938                     )
939                 ):
940                     # Never omit bracket pairs with trailing commas.
941                     # We need to explode on those.
942                     break
943
944                 inner_brackets.add(id(leaf))
945         elif leaf.type in CLOSING_BRACKETS:
946             prev = line.leaves[index - 1] if index > 0 else None
947             if prev and prev.type in OPENING_BRACKETS:
948                 # Empty brackets would fail a split so treat them as "inner"
949                 # brackets (e.g. only add them to the `omit` set if another
950                 # pair of brackets was good enough.
951                 inner_brackets.add(id(leaf))
952                 continue
953
954             if closing_bracket:
955                 omit.add(id(closing_bracket))
956                 omit.update(inner_brackets)
957                 inner_brackets.clear()
958                 yield omit
959
960             if (
961                 prev
962                 and prev.type == token.COMMA
963                 and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves)
964             ):
965                 # Never omit bracket pairs with trailing commas.
966                 # We need to explode on those.
967                 break
968
969             if leaf.value:
970                 opening_bracket = leaf.opening_bracket
971                 closing_bracket = leaf
972
973
974 def run_transformer(
975     line: Line,
976     transform: Transformer,
977     mode: Mode,
978     features: Collection[Feature],
979     *,
980     line_str: str = "",
981 ) -> List[Line]:
982     if not line_str:
983         line_str = line_to_string(line)
984     result: List[Line] = []
985     for transformed_line in transform(line, features):
986         if str(transformed_line).strip("\n") == line_str:
987             raise CannotTransform("Line transformer returned an unchanged result")
988
989         result.extend(transform_line(transformed_line, mode=mode, features=features))
990
991     if (
992         transform.__class__.__name__ != "rhs"
993         or not line.bracket_tracker.invisible
994         or any(bracket.value for bracket in line.bracket_tracker.invisible)
995         or line.contains_multiline_strings()
996         or result[0].contains_uncollapsable_type_comments()
997         or result[0].contains_unsplittable_type_ignore()
998         or is_line_short_enough(result[0], line_length=mode.line_length)
999         # If any leaves have no parents (which _can_ occur since
1000         # `transform(line)` potentially destroys the line's underlying node
1001         # structure), then we can't proceed. Doing so would cause the below
1002         # call to `append_leaves()` to fail.
1003         or any(leaf.parent is None for leaf in line.leaves)
1004     ):
1005         return result
1006
1007     line_copy = line.clone()
1008     append_leaves(line_copy, line, line.leaves)
1009     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1010     second_opinion = run_transformer(
1011         line_copy, transform, mode, features_fop, line_str=line_str
1012     )
1013     if all(
1014         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1015     ):
1016         result = second_opinion
1017     return result