]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

no longer expect changes on pyanalyze (#2674)
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 from functools import partial, wraps
5 import sys
6 from typing import Collection, Iterator, List, Optional, Set, Union
7
8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
10 from black.nodes import Visitor, syms, first_child_is_arith, ensure_visible
11 from black.nodes import is_docstring, is_empty_tuple, is_one_tuple, is_one_tuple_between
12 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
13 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
14 from black.nodes import wrap_in_parentheses
15 from black.brackets import max_delimiter_priority_in_atom
16 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
17 from black.lines import Line, line_to_string, is_line_short_enough
18 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
19 from black.comments import generate_comments, list_comments, FMT_OFF
20 from black.numerics import normalize_numeric_literal
21 from black.strings import get_string_prefix, fix_docstring
22 from black.strings import normalize_string_prefix, normalize_string_quotes
23 from black.trans import Transformer, CannotTransform, StringMerger
24 from black.trans import StringSplitter, StringParenWrapper, StringParenStripper
25 from black.mode import Mode
26 from black.mode import Feature
27
28 from blib2to3.pytree import Node, Leaf
29 from blib2to3.pgen2 import token
30
31
32 # types
33 LeafID = int
34 LN = Union[Leaf, Node]
35
36
37 class CannotSplit(CannotTransform):
38     """A readable split that fits the allotted line length is impossible."""
39
40
41 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
42 # See also https://github.com/mypyc/mypyc/issues/827.
43 class LineGenerator(Visitor[Line]):
44     """Generates reformatted Line objects.  Empty lines are not emitted.
45
46     Note: destroys the tree it's visiting by mutating prefixes of its leaves
47     in ways that will no longer stringify to valid Python code on the tree.
48     """
49
50     def __init__(self, mode: Mode, remove_u_prefix: bool = False) -> None:
51         self.mode = mode
52         self.remove_u_prefix = remove_u_prefix
53         self.current_line: Line
54         self.__post_init__()
55
56     def line(self, indent: int = 0) -> Iterator[Line]:
57         """Generate a line.
58
59         If the line is empty, only emit if it makes sense.
60         If the line is too long, split it first and then generate.
61
62         If any lines were generated, set up a new current_line.
63         """
64         if not self.current_line:
65             self.current_line.depth += indent
66             return  # Line is empty, don't emit. Creating a new one unnecessary.
67
68         complete_line = self.current_line
69         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
70         yield complete_line
71
72     def visit_default(self, node: LN) -> Iterator[Line]:
73         """Default `visit_*()` implementation. Recurses to children of `node`."""
74         if isinstance(node, Leaf):
75             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
76             for comment in generate_comments(node):
77                 if any_open_brackets:
78                     # any comment within brackets is subject to splitting
79                     self.current_line.append(comment)
80                 elif comment.type == token.COMMENT:
81                     # regular trailing comment
82                     self.current_line.append(comment)
83                     yield from self.line()
84
85                 else:
86                     # regular standalone comment
87                     yield from self.line()
88
89                     self.current_line.append(comment)
90                     yield from self.line()
91
92             normalize_prefix(node, inside_brackets=any_open_brackets)
93             if self.mode.string_normalization and node.type == token.STRING:
94                 node.value = normalize_string_prefix(
95                     node.value, remove_u_prefix=self.remove_u_prefix
96                 )
97                 node.value = normalize_string_quotes(node.value)
98             if node.type == token.NUMBER:
99                 normalize_numeric_literal(node)
100             if node.type not in WHITESPACE:
101                 self.current_line.append(node)
102         yield from super().visit_default(node)
103
104     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
105         """Increase indentation level, maybe yield a line."""
106         # In blib2to3 INDENT never holds comments.
107         yield from self.line(+1)
108         yield from self.visit_default(node)
109
110     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
111         """Decrease indentation level, maybe yield a line."""
112         # The current line might still wait for trailing comments.  At DEDENT time
113         # there won't be any (they would be prefixes on the preceding NEWLINE).
114         # Emit the line then.
115         yield from self.line()
116
117         # While DEDENT has no value, its prefix may contain standalone comments
118         # that belong to the current indentation level.  Get 'em.
119         yield from self.visit_default(node)
120
121         # Finally, emit the dedent.
122         yield from self.line(-1)
123
124     def visit_stmt(
125         self, node: Node, keywords: Set[str], parens: Set[str]
126     ) -> Iterator[Line]:
127         """Visit a statement.
128
129         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
130         `def`, `with`, `class`, `assert`, and assignments.
131
132         The relevant Python language `keywords` for a given statement will be
133         NAME leaves within it. This methods puts those on a separate line.
134
135         `parens` holds a set of string leaf values immediately after which
136         invisible parens should be put.
137         """
138         normalize_invisible_parens(node, parens_after=parens)
139         for child in node.children:
140             if child.type == token.NAME and child.value in keywords:  # type: ignore
141                 yield from self.line()
142
143             yield from self.visit(child)
144
145     def visit_match_case(self, node: Node) -> Iterator[Line]:
146         """Visit either a match or case statement."""
147         normalize_invisible_parens(node, parens_after=set())
148
149         yield from self.line()
150         for child in node.children:
151             yield from self.visit(child)
152
153     def visit_suite(self, node: Node) -> Iterator[Line]:
154         """Visit a suite."""
155         if self.mode.is_pyi and is_stub_suite(node):
156             yield from self.visit(node.children[2])
157         else:
158             yield from self.visit_default(node)
159
160     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
161         """Visit a statement without nested statements."""
162         if first_child_is_arith(node):
163             wrap_in_parentheses(node, node.children[0], visible=False)
164         is_suite_like = node.parent and node.parent.type in STATEMENT
165         if is_suite_like:
166             if self.mode.is_pyi and is_stub_body(node):
167                 yield from self.visit_default(node)
168             else:
169                 yield from self.line(+1)
170                 yield from self.visit_default(node)
171                 yield from self.line(-1)
172
173         else:
174             if (
175                 not self.mode.is_pyi
176                 or not node.parent
177                 or not is_stub_suite(node.parent)
178             ):
179                 yield from self.line()
180             yield from self.visit_default(node)
181
182     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
183         """Visit `async def`, `async for`, `async with`."""
184         yield from self.line()
185
186         children = iter(node.children)
187         for child in children:
188             yield from self.visit(child)
189
190             if child.type == token.ASYNC:
191                 break
192
193         internal_stmt = next(children)
194         for child in internal_stmt.children:
195             yield from self.visit(child)
196
197     def visit_decorators(self, node: Node) -> Iterator[Line]:
198         """Visit decorators."""
199         for child in node.children:
200             yield from self.line()
201             yield from self.visit(child)
202
203     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
204         """Remove a semicolon and put the other statement on a separate line."""
205         yield from self.line()
206
207     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
208         """End of file. Process outstanding comments and end with a newline."""
209         yield from self.visit_default(leaf)
210         yield from self.line()
211
212     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
213         if not self.current_line.bracket_tracker.any_open_brackets():
214             yield from self.line()
215         yield from self.visit_default(leaf)
216
217     def visit_factor(self, node: Node) -> Iterator[Line]:
218         """Force parentheses between a unary op and a binary power:
219
220         -2 ** 8 -> -(2 ** 8)
221         """
222         _operator, operand = node.children
223         if (
224             operand.type == syms.power
225             and len(operand.children) == 3
226             and operand.children[1].type == token.DOUBLESTAR
227         ):
228             lpar = Leaf(token.LPAR, "(")
229             rpar = Leaf(token.RPAR, ")")
230             index = operand.remove() or 0
231             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
232         yield from self.visit_default(node)
233
234     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
235         if is_docstring(leaf) and "\\\n" not in leaf.value:
236             # We're ignoring docstrings with backslash newline escapes because changing
237             # indentation of those changes the AST representation of the code.
238             docstring = normalize_string_prefix(leaf.value, self.remove_u_prefix)
239             prefix = get_string_prefix(docstring)
240             docstring = docstring[len(prefix) :]  # Remove the prefix
241             quote_char = docstring[0]
242             # A natural way to remove the outer quotes is to do:
243             #   docstring = docstring.strip(quote_char)
244             # but that breaks on """""x""" (which is '""x').
245             # So we actually need to remove the first character and the next two
246             # characters but only if they are the same as the first.
247             quote_len = 1 if docstring[1] != quote_char else 3
248             docstring = docstring[quote_len:-quote_len]
249             docstring_started_empty = not docstring
250
251             if is_multiline_string(leaf):
252                 indent = " " * 4 * self.current_line.depth
253                 docstring = fix_docstring(docstring, indent)
254             else:
255                 docstring = docstring.strip()
256
257             if docstring:
258                 # Add some padding if the docstring starts / ends with a quote mark.
259                 if docstring[0] == quote_char:
260                     docstring = " " + docstring
261                 if docstring[-1] == quote_char:
262                     docstring += " "
263                 if docstring[-1] == "\\":
264                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
265                     if backslash_count % 2:
266                         # Odd number of tailing backslashes, add some padding to
267                         # avoid escaping the closing string quote.
268                         docstring += " "
269             elif not docstring_started_empty:
270                 docstring = " "
271
272             # We could enforce triple quotes at this point.
273             quote = quote_char * quote_len
274             leaf.value = prefix + quote + docstring + quote
275
276         yield from self.visit_default(leaf)
277
278     def __post_init__(self) -> None:
279         """You are in a twisty little maze of passages."""
280         self.current_line = Line(mode=self.mode)
281
282         v = self.visit_stmt
283         Ø: Set[str] = set()
284         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
285         self.visit_if_stmt = partial(
286             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
287         )
288         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
289         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
290         self.visit_try_stmt = partial(
291             v, keywords={"try", "except", "else", "finally"}, parens=Ø
292         )
293         self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
294         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
295         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
296         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
297         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
298         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
299         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
300         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
301         self.visit_async_funcdef = self.visit_async_stmt
302         self.visit_decorated = self.visit_decorators
303
304         # PEP 634
305         self.visit_match_stmt = self.visit_match_case
306         self.visit_case_block = self.visit_match_case
307
308
309 def transform_line(
310     line: Line, mode: Mode, features: Collection[Feature] = ()
311 ) -> Iterator[Line]:
312     """Transform a `line`, potentially splitting it into many lines.
313
314     They should fit in the allotted `line_length` but might not be able to.
315
316     `features` are syntactical features that may be used in the output.
317     """
318     if line.is_comment:
319         yield line
320         return
321
322     line_str = line_to_string(line)
323
324     ll = mode.line_length
325     sn = mode.string_normalization
326     string_merge = StringMerger(ll, sn)
327     string_paren_strip = StringParenStripper(ll, sn)
328     string_split = StringSplitter(ll, sn)
329     string_paren_wrap = StringParenWrapper(ll, sn)
330
331     transformers: List[Transformer]
332     if (
333         not line.contains_uncollapsable_type_comments()
334         and not line.should_split_rhs
335         and not line.magic_trailing_comma
336         and (
337             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
338             or line.contains_unsplittable_type_ignore()
339         )
340         and not (line.inside_brackets and line.contains_standalone_comments())
341     ):
342         # Only apply basic string preprocessing, since lines shouldn't be split here.
343         if mode.experimental_string_processing:
344             transformers = [string_merge, string_paren_strip]
345         else:
346             transformers = []
347     elif line.is_def:
348         transformers = [left_hand_split]
349     else:
350
351         def _rhs(
352             self: object, line: Line, features: Collection[Feature]
353         ) -> Iterator[Line]:
354             """Wraps calls to `right_hand_split`.
355
356             The calls increasingly `omit` right-hand trailers (bracket pairs with
357             content), meaning the trailers get glued together to split on another
358             bracket pair instead.
359             """
360             for omit in generate_trailers_to_omit(line, mode.line_length):
361                 lines = list(
362                     right_hand_split(line, mode.line_length, features, omit=omit)
363                 )
364                 # Note: this check is only able to figure out if the first line of the
365                 # *current* transformation fits in the line length.  This is true only
366                 # for simple cases.  All others require running more transforms via
367                 # `transform_line()`.  This check doesn't know if those would succeed.
368                 if is_line_short_enough(lines[0], line_length=mode.line_length):
369                     yield from lines
370                     return
371
372             # All splits failed, best effort split with no omits.
373             # This mostly happens to multiline strings that are by definition
374             # reported as not fitting a single line, as well as lines that contain
375             # trailing commas (those have to be exploded).
376             yield from right_hand_split(
377                 line, line_length=mode.line_length, features=features
378             )
379
380         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
381         # __name__ attribute which is needed in `run_transformer` further down.
382         # Unfortunately a nested class breaks mypyc too. So a class must be created
383         # via type ... https://github.com/mypyc/mypyc/issues/884
384         rhs = type("rhs", (), {"__call__": _rhs})()
385
386         if mode.experimental_string_processing:
387             if line.inside_brackets:
388                 transformers = [
389                     string_merge,
390                     string_paren_strip,
391                     string_split,
392                     delimiter_split,
393                     standalone_comment_split,
394                     string_paren_wrap,
395                     rhs,
396                 ]
397             else:
398                 transformers = [
399                     string_merge,
400                     string_paren_strip,
401                     string_split,
402                     string_paren_wrap,
403                     rhs,
404                 ]
405         else:
406             if line.inside_brackets:
407                 transformers = [delimiter_split, standalone_comment_split, rhs]
408             else:
409                 transformers = [rhs]
410
411     for transform in transformers:
412         # We are accumulating lines in `result` because we might want to abort
413         # mission and return the original line in the end, or attempt a different
414         # split altogether.
415         try:
416             result = run_transformer(line, transform, mode, features, line_str=line_str)
417         except CannotTransform:
418             continue
419         else:
420             yield from result
421             break
422
423     else:
424         yield line
425
426
427 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
428     """Split line into many lines, starting with the first matching bracket pair.
429
430     Note: this usually looks weird, only use this for function definitions.
431     Prefer RHS otherwise.  This is why this function is not symmetrical with
432     :func:`right_hand_split` which also handles optional parentheses.
433     """
434     tail_leaves: List[Leaf] = []
435     body_leaves: List[Leaf] = []
436     head_leaves: List[Leaf] = []
437     current_leaves = head_leaves
438     matching_bracket: Optional[Leaf] = None
439     for leaf in line.leaves:
440         if (
441             current_leaves is body_leaves
442             and leaf.type in CLOSING_BRACKETS
443             and leaf.opening_bracket is matching_bracket
444         ):
445             current_leaves = tail_leaves if body_leaves else head_leaves
446         current_leaves.append(leaf)
447         if current_leaves is head_leaves:
448             if leaf.type in OPENING_BRACKETS:
449                 matching_bracket = leaf
450                 current_leaves = body_leaves
451     if not matching_bracket:
452         raise CannotSplit("No brackets found")
453
454     head = bracket_split_build_line(head_leaves, line, matching_bracket)
455     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
456     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
457     bracket_split_succeeded_or_raise(head, body, tail)
458     for result in (head, body, tail):
459         if result:
460             yield result
461
462
463 def right_hand_split(
464     line: Line,
465     line_length: int,
466     features: Collection[Feature] = (),
467     omit: Collection[LeafID] = (),
468 ) -> Iterator[Line]:
469     """Split line into many lines, starting with the last matching bracket pair.
470
471     If the split was by optional parentheses, attempt splitting without them, too.
472     `omit` is a collection of closing bracket IDs that shouldn't be considered for
473     this split.
474
475     Note: running this function modifies `bracket_depth` on the leaves of `line`.
476     """
477     tail_leaves: List[Leaf] = []
478     body_leaves: List[Leaf] = []
479     head_leaves: List[Leaf] = []
480     current_leaves = tail_leaves
481     opening_bracket: Optional[Leaf] = None
482     closing_bracket: Optional[Leaf] = None
483     for leaf in reversed(line.leaves):
484         if current_leaves is body_leaves:
485             if leaf is opening_bracket:
486                 current_leaves = head_leaves if body_leaves else tail_leaves
487         current_leaves.append(leaf)
488         if current_leaves is tail_leaves:
489             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
490                 opening_bracket = leaf.opening_bracket
491                 closing_bracket = leaf
492                 current_leaves = body_leaves
493     if not (opening_bracket and closing_bracket and head_leaves):
494         # If there is no opening or closing_bracket that means the split failed and
495         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
496         # the matching `opening_bracket` wasn't available on `line` anymore.
497         raise CannotSplit("No brackets found")
498
499     tail_leaves.reverse()
500     body_leaves.reverse()
501     head_leaves.reverse()
502     head = bracket_split_build_line(head_leaves, line, opening_bracket)
503     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
504     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
505     bracket_split_succeeded_or_raise(head, body, tail)
506     if (
507         Feature.FORCE_OPTIONAL_PARENTHESES not in features
508         # the opening bracket is an optional paren
509         and opening_bracket.type == token.LPAR
510         and not opening_bracket.value
511         # the closing bracket is an optional paren
512         and closing_bracket.type == token.RPAR
513         and not closing_bracket.value
514         # it's not an import (optional parens are the only thing we can split on
515         # in this case; attempting a split without them is a waste of time)
516         and not line.is_import
517         # there are no standalone comments in the body
518         and not body.contains_standalone_comments(0)
519         # and we can actually remove the parens
520         and can_omit_invisible_parens(body, line_length, omit_on_explode=omit)
521     ):
522         omit = {id(closing_bracket), *omit}
523         try:
524             yield from right_hand_split(line, line_length, features=features, omit=omit)
525             return
526
527         except CannotSplit as e:
528             if not (
529                 can_be_split(body)
530                 or is_line_short_enough(body, line_length=line_length)
531             ):
532                 raise CannotSplit(
533                     "Splitting failed, body is still too long and can't be split."
534                 ) from e
535
536             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
537                 raise CannotSplit(
538                     "The current optional pair of parentheses is bound to fail to"
539                     " satisfy the splitting algorithm because the head or the tail"
540                     " contains multiline strings which by definition never fit one"
541                     " line."
542                 ) from e
543
544     ensure_visible(opening_bracket)
545     ensure_visible(closing_bracket)
546     for result in (head, body, tail):
547         if result:
548             yield result
549
550
551 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
552     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
553
554     Do nothing otherwise.
555
556     A left- or right-hand split is based on a pair of brackets. Content before
557     (and including) the opening bracket is left on one line, content inside the
558     brackets is put on a separate line, and finally content starting with and
559     following the closing bracket is put on a separate line.
560
561     Those are called `head`, `body`, and `tail`, respectively. If the split
562     produced the same line (all content in `head`) or ended up with an empty `body`
563     and the `tail` is just the closing bracket, then it's considered failed.
564     """
565     tail_len = len(str(tail).strip())
566     if not body:
567         if tail_len == 0:
568             raise CannotSplit("Splitting brackets produced the same line")
569
570         elif tail_len < 3:
571             raise CannotSplit(
572                 f"Splitting brackets on an empty body to save {tail_len} characters is"
573                 " not worth it"
574             )
575
576
577 def bracket_split_build_line(
578     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
579 ) -> Line:
580     """Return a new line with given `leaves` and respective comments from `original`.
581
582     If `is_body` is True, the result line is one-indented inside brackets and as such
583     has its first leaf's prefix normalized and a trailing comma added when expected.
584     """
585     result = Line(mode=original.mode, depth=original.depth)
586     if is_body:
587         result.inside_brackets = True
588         result.depth += 1
589         if leaves:
590             # Since body is a new indent level, remove spurious leading whitespace.
591             normalize_prefix(leaves[0], inside_brackets=True)
592             # Ensure a trailing comma for imports and standalone function arguments, but
593             # be careful not to add one after any comments or within type annotations.
594             no_commas = (
595                 original.is_def
596                 and opening_bracket.value == "("
597                 and not any(leaf.type == token.COMMA for leaf in leaves)
598                 # In particular, don't add one within a parenthesized return annotation.
599                 # Unfortunately the indicator we're in a return annotation (RARROW) may
600                 # be defined directly in the parent node, the parent of the parent ...
601                 # and so on depending on how complex the return annotation is.
602                 # This isn't perfect and there's some false negatives but they are in
603                 # contexts were a comma is actually fine.
604                 and not any(
605                     node.prev_sibling.type == RARROW
606                     for node in (
607                         leaves[0].parent,
608                         getattr(leaves[0].parent, "parent", None),
609                     )
610                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
611                 )
612             )
613
614             if original.is_import or no_commas:
615                 for i in range(len(leaves) - 1, -1, -1):
616                     if leaves[i].type == STANDALONE_COMMENT:
617                         continue
618
619                     if leaves[i].type != token.COMMA:
620                         new_comma = Leaf(token.COMMA, ",")
621                         leaves.insert(i + 1, new_comma)
622                     break
623
624     # Populate the line
625     for leaf in leaves:
626         result.append(leaf, preformatted=True)
627         for comment_after in original.comments_after(leaf):
628             result.append(comment_after, preformatted=True)
629     if is_body and should_split_line(result, opening_bracket):
630         result.should_split_rhs = True
631     return result
632
633
634 def dont_increase_indentation(split_func: Transformer) -> Transformer:
635     """Normalize prefix of the first leaf in every line returned by `split_func`.
636
637     This is a decorator over relevant split functions.
638     """
639
640     @wraps(split_func)
641     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
642         for line in split_func(line, features):
643             normalize_prefix(line.leaves[0], inside_brackets=True)
644             yield line
645
646     return split_wrapper
647
648
649 @dont_increase_indentation
650 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
651     """Split according to delimiters of the highest priority.
652
653     If the appropriate Features are given, the split will add trailing commas
654     also in function signatures and calls that contain `*` and `**`.
655     """
656     try:
657         last_leaf = line.leaves[-1]
658     except IndexError:
659         raise CannotSplit("Line empty") from None
660
661     bt = line.bracket_tracker
662     try:
663         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
664     except ValueError:
665         raise CannotSplit("No delimiters found") from None
666
667     if delimiter_priority == DOT_PRIORITY:
668         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
669             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
670
671     current_line = Line(
672         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
673     )
674     lowest_depth = sys.maxsize
675     trailing_comma_safe = True
676
677     def append_to_line(leaf: Leaf) -> Iterator[Line]:
678         """Append `leaf` to current line or to new line if appending impossible."""
679         nonlocal current_line
680         try:
681             current_line.append_safe(leaf, preformatted=True)
682         except ValueError:
683             yield current_line
684
685             current_line = Line(
686                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
687             )
688             current_line.append(leaf)
689
690     for leaf in line.leaves:
691         yield from append_to_line(leaf)
692
693         for comment_after in line.comments_after(leaf):
694             yield from append_to_line(comment_after)
695
696         lowest_depth = min(lowest_depth, leaf.bracket_depth)
697         if leaf.bracket_depth == lowest_depth:
698             if is_vararg(leaf, within={syms.typedargslist}):
699                 trailing_comma_safe = (
700                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
701                 )
702             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
703                 trailing_comma_safe = (
704                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
705                 )
706
707         leaf_priority = bt.delimiters.get(id(leaf))
708         if leaf_priority == delimiter_priority:
709             yield current_line
710
711             current_line = Line(
712                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
713             )
714     if current_line:
715         if (
716             trailing_comma_safe
717             and delimiter_priority == COMMA_PRIORITY
718             and current_line.leaves[-1].type != token.COMMA
719             and current_line.leaves[-1].type != STANDALONE_COMMENT
720         ):
721             new_comma = Leaf(token.COMMA, ",")
722             current_line.append(new_comma)
723         yield current_line
724
725
726 @dont_increase_indentation
727 def standalone_comment_split(
728     line: Line, features: Collection[Feature] = ()
729 ) -> Iterator[Line]:
730     """Split standalone comments from the rest of the line."""
731     if not line.contains_standalone_comments(0):
732         raise CannotSplit("Line does not have any standalone comments")
733
734     current_line = Line(
735         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
736     )
737
738     def append_to_line(leaf: Leaf) -> Iterator[Line]:
739         """Append `leaf` to current line or to new line if appending impossible."""
740         nonlocal current_line
741         try:
742             current_line.append_safe(leaf, preformatted=True)
743         except ValueError:
744             yield current_line
745
746             current_line = Line(
747                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
748             )
749             current_line.append(leaf)
750
751     for leaf in line.leaves:
752         yield from append_to_line(leaf)
753
754         for comment_after in line.comments_after(leaf):
755             yield from append_to_line(comment_after)
756
757     if current_line:
758         yield current_line
759
760
761 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
762     """Leave existing extra newlines if not `inside_brackets`. Remove everything
763     else.
764
765     Note: don't use backslashes for formatting or you'll lose your voting rights.
766     """
767     if not inside_brackets:
768         spl = leaf.prefix.split("#")
769         if "\\" not in spl[0]:
770             nl_count = spl[-1].count("\n")
771             if len(spl) > 1:
772                 nl_count -= 1
773             leaf.prefix = "\n" * nl_count
774             return
775
776     leaf.prefix = ""
777
778
779 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
780     """Make existing optional parentheses invisible or create new ones.
781
782     `parens_after` is a set of string leaf values immediately after which parens
783     should be put.
784
785     Standardizes on visible parentheses for single-element tuples, and keeps
786     existing visible parentheses for other tuples and generator expressions.
787     """
788     for pc in list_comments(node.prefix, is_endmarker=False):
789         if pc.value in FMT_OFF:
790             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
791             return
792     check_lpar = False
793     for index, child in enumerate(list(node.children)):
794         # Fixes a bug where invisible parens are not properly stripped from
795         # assignment statements that contain type annotations.
796         if isinstance(child, Node) and child.type == syms.annassign:
797             normalize_invisible_parens(child, parens_after=parens_after)
798
799         # Add parentheses around long tuple unpacking in assignments.
800         if (
801             index == 0
802             and isinstance(child, Node)
803             and child.type == syms.testlist_star_expr
804         ):
805             check_lpar = True
806
807         if check_lpar:
808             if child.type == syms.atom:
809                 if maybe_make_parens_invisible_in_atom(child, parent=node):
810                     wrap_in_parentheses(node, child, visible=False)
811             elif is_one_tuple(child):
812                 wrap_in_parentheses(node, child, visible=True)
813             elif node.type == syms.import_from:
814                 # "import from" nodes store parentheses directly as part of
815                 # the statement
816                 if child.type == token.LPAR:
817                     # make parentheses invisible
818                     child.value = ""  # type: ignore
819                     node.children[-1].value = ""  # type: ignore
820                 elif child.type != token.STAR:
821                     # insert invisible parentheses
822                     node.insert_child(index, Leaf(token.LPAR, ""))
823                     node.append_child(Leaf(token.RPAR, ""))
824                 break
825
826             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
827                 wrap_in_parentheses(node, child, visible=False)
828
829         check_lpar = isinstance(child, Leaf) and child.value in parens_after
830
831
832 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
833     """If it's safe, make the parens in the atom `node` invisible, recursively.
834     Additionally, remove repeated, adjacent invisible parens from the atom `node`
835     as they are redundant.
836
837     Returns whether the node should itself be wrapped in invisible parentheses.
838
839     """
840
841     if (
842         node.type != syms.atom
843         or is_empty_tuple(node)
844         or is_one_tuple(node)
845         or (is_yield(node) and parent.type != syms.expr_stmt)
846         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
847     ):
848         return False
849
850     if is_walrus_assignment(node):
851         if parent.type in [
852             syms.annassign,
853             syms.expr_stmt,
854             syms.assert_stmt,
855             syms.return_stmt,
856             # these ones aren't useful to end users, but they do please fuzzers
857             syms.for_stmt,
858             syms.del_stmt,
859         ]:
860             return False
861
862     first = node.children[0]
863     last = node.children[-1]
864     if first.type == token.LPAR and last.type == token.RPAR:
865         middle = node.children[1]
866         # make parentheses invisible
867         first.value = ""  # type: ignore
868         last.value = ""  # type: ignore
869         maybe_make_parens_invisible_in_atom(middle, parent=parent)
870
871         if is_atom_with_invisible_parens(middle):
872             # Strip the invisible parens from `middle` by replacing
873             # it with the child in-between the invisible parens
874             middle.replace(middle.children[1])
875
876         return False
877
878     return True
879
880
881 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
882     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
883
884     if not (opening_bracket.parent and opening_bracket.value in "[{("):
885         return False
886
887     # We're essentially checking if the body is delimited by commas and there's more
888     # than one of them (we're excluding the trailing comma and if the delimiter priority
889     # is still commas, that means there's more).
890     exclude = set()
891     trailing_comma = False
892     try:
893         last_leaf = line.leaves[-1]
894         if last_leaf.type == token.COMMA:
895             trailing_comma = True
896             exclude.add(id(last_leaf))
897         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
898     except (IndexError, ValueError):
899         return False
900
901     return max_priority == COMMA_PRIORITY and (
902         (line.mode.magic_trailing_comma and trailing_comma)
903         # always explode imports
904         or opening_bracket.parent.type in {syms.atom, syms.import_from}
905     )
906
907
908 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
909     """Generate sets of closing bracket IDs that should be omitted in a RHS.
910
911     Brackets can be omitted if the entire trailer up to and including
912     a preceding closing bracket fits in one line.
913
914     Yielded sets are cumulative (contain results of previous yields, too).  First
915     set is empty, unless the line should explode, in which case bracket pairs until
916     the one that needs to explode are omitted.
917     """
918
919     omit: Set[LeafID] = set()
920     if not line.magic_trailing_comma:
921         yield omit
922
923     length = 4 * line.depth
924     opening_bracket: Optional[Leaf] = None
925     closing_bracket: Optional[Leaf] = None
926     inner_brackets: Set[LeafID] = set()
927     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
928         length += leaf_length
929         if length > line_length:
930             break
931
932         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
933         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
934             break
935
936         if opening_bracket:
937             if leaf is opening_bracket:
938                 opening_bracket = None
939             elif leaf.type in CLOSING_BRACKETS:
940                 prev = line.leaves[index - 1] if index > 0 else None
941                 if (
942                     prev
943                     and prev.type == token.COMMA
944                     and not is_one_tuple_between(
945                         leaf.opening_bracket, leaf, line.leaves
946                     )
947                 ):
948                     # Never omit bracket pairs with trailing commas.
949                     # We need to explode on those.
950                     break
951
952                 inner_brackets.add(id(leaf))
953         elif leaf.type in CLOSING_BRACKETS:
954             prev = line.leaves[index - 1] if index > 0 else None
955             if prev and prev.type in OPENING_BRACKETS:
956                 # Empty brackets would fail a split so treat them as "inner"
957                 # brackets (e.g. only add them to the `omit` set if another
958                 # pair of brackets was good enough.
959                 inner_brackets.add(id(leaf))
960                 continue
961
962             if closing_bracket:
963                 omit.add(id(closing_bracket))
964                 omit.update(inner_brackets)
965                 inner_brackets.clear()
966                 yield omit
967
968             if (
969                 prev
970                 and prev.type == token.COMMA
971                 and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves)
972             ):
973                 # Never omit bracket pairs with trailing commas.
974                 # We need to explode on those.
975                 break
976
977             if leaf.value:
978                 opening_bracket = leaf.opening_bracket
979                 closing_bracket = leaf
980
981
982 def run_transformer(
983     line: Line,
984     transform: Transformer,
985     mode: Mode,
986     features: Collection[Feature],
987     *,
988     line_str: str = "",
989 ) -> List[Line]:
990     if not line_str:
991         line_str = line_to_string(line)
992     result: List[Line] = []
993     for transformed_line in transform(line, features):
994         if str(transformed_line).strip("\n") == line_str:
995             raise CannotTransform("Line transformer returned an unchanged result")
996
997         result.extend(transform_line(transformed_line, mode=mode, features=features))
998
999     if (
1000         transform.__class__.__name__ != "rhs"
1001         or not line.bracket_tracker.invisible
1002         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1003         or line.contains_multiline_strings()
1004         or result[0].contains_uncollapsable_type_comments()
1005         or result[0].contains_unsplittable_type_ignore()
1006         or is_line_short_enough(result[0], line_length=mode.line_length)
1007         # If any leaves have no parents (which _can_ occur since
1008         # `transform(line)` potentially destroys the line's underlying node
1009         # structure), then we can't proceed. Doing so would cause the below
1010         # call to `append_leaves()` to fail.
1011         or any(leaf.parent is None for leaf in line.leaves)
1012     ):
1013         return result
1014
1015     line_copy = line.clone()
1016     append_leaves(line_copy, line, line.leaves)
1017     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1018     second_opinion = run_transformer(
1019         line_copy, transform, mode, features_fop, line_str=line_str
1020     )
1021     if all(
1022         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1023     ):
1024         result = second_opinion
1025     return result