]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Bump pypa/cibuildwheel from 2.13.0 to 2.13.1 (#3729)
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 import sys
5 from dataclasses import replace
6 from enum import Enum, auto
7 from functools import partial, wraps
8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
9
10 from black.brackets import (
11     COMMA_PRIORITY,
12     DOT_PRIORITY,
13     get_leaves_inside_matching_brackets,
14     max_delimiter_priority_in_atom,
15 )
16 from black.comments import FMT_OFF, generate_comments, list_comments
17 from black.lines import (
18     Line,
19     RHSResult,
20     append_leaves,
21     can_be_split,
22     can_omit_invisible_parens,
23     is_line_short_enough,
24     line_to_string,
25 )
26 from black.mode import Feature, Mode, Preview
27 from black.nodes import (
28     ASSIGNMENTS,
29     BRACKETS,
30     CLOSING_BRACKETS,
31     OPENING_BRACKETS,
32     RARROW,
33     STANDALONE_COMMENT,
34     STATEMENT,
35     WHITESPACE,
36     Visitor,
37     ensure_visible,
38     is_arith_like,
39     is_async_stmt_or_funcdef,
40     is_atom_with_invisible_parens,
41     is_docstring,
42     is_empty_tuple,
43     is_lpar_token,
44     is_multiline_string,
45     is_name_token,
46     is_one_sequence_between,
47     is_one_tuple,
48     is_rpar_token,
49     is_stub_body,
50     is_stub_suite,
51     is_tuple_containing_walrus,
52     is_vararg,
53     is_walrus_assignment,
54     is_yield,
55     syms,
56     wrap_in_parentheses,
57 )
58 from black.numerics import normalize_numeric_literal
59 from black.strings import (
60     fix_docstring,
61     get_string_prefix,
62     normalize_string_prefix,
63     normalize_string_quotes,
64     normalize_unicode_escape_sequences,
65 )
66 from black.trans import (
67     CannotTransform,
68     StringMerger,
69     StringParenStripper,
70     StringParenWrapper,
71     StringSplitter,
72     Transformer,
73     hug_power_op,
74 )
75 from blib2to3.pgen2 import token
76 from blib2to3.pytree import Leaf, Node
77
78 # types
79 LeafID = int
80 LN = Union[Leaf, Node]
81
82
83 class CannotSplit(CannotTransform):
84     """A readable split that fits the allotted line length is impossible."""
85
86
87 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
88 # See also https://github.com/mypyc/mypyc/issues/827.
89 class LineGenerator(Visitor[Line]):
90     """Generates reformatted Line objects.  Empty lines are not emitted.
91
92     Note: destroys the tree it's visiting by mutating prefixes of its leaves
93     in ways that will no longer stringify to valid Python code on the tree.
94     """
95
96     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
97         self.mode = mode
98         self.features = features
99         self.current_line: Line
100         self.__post_init__()
101
102     def line(self, indent: int = 0) -> Iterator[Line]:
103         """Generate a line.
104
105         If the line is empty, only emit if it makes sense.
106         If the line is too long, split it first and then generate.
107
108         If any lines were generated, set up a new current_line.
109         """
110         if not self.current_line:
111             self.current_line.depth += indent
112             return  # Line is empty, don't emit. Creating a new one unnecessary.
113
114         if (
115             Preview.improved_async_statements_handling in self.mode
116             and len(self.current_line.leaves) == 1
117             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
118         ):
119             # Special case for async def/for/with statements. `visit_async_stmt`
120             # adds an `ASYNC` leaf then visits the child def/for/with statement
121             # nodes. Line yields from those nodes shouldn't treat the former
122             # `ASYNC` leaf as a complete line.
123             return
124
125         complete_line = self.current_line
126         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
127         yield complete_line
128
129     def visit_default(self, node: LN) -> Iterator[Line]:
130         """Default `visit_*()` implementation. Recurses to children of `node`."""
131         if isinstance(node, Leaf):
132             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
133             for comment in generate_comments(node):
134                 if any_open_brackets:
135                     # any comment within brackets is subject to splitting
136                     self.current_line.append(comment)
137                 elif comment.type == token.COMMENT:
138                     # regular trailing comment
139                     self.current_line.append(comment)
140                     yield from self.line()
141
142                 else:
143                     # regular standalone comment
144                     yield from self.line()
145
146                     self.current_line.append(comment)
147                     yield from self.line()
148
149             normalize_prefix(node, inside_brackets=any_open_brackets)
150             if self.mode.string_normalization and node.type == token.STRING:
151                 node.value = normalize_string_prefix(node.value)
152                 node.value = normalize_string_quotes(node.value)
153             if node.type == token.NUMBER:
154                 normalize_numeric_literal(node)
155             if node.type not in WHITESPACE:
156                 self.current_line.append(node)
157         yield from super().visit_default(node)
158
159     def visit_test(self, node: Node) -> Iterator[Line]:
160         """Visit an `x if y else z` test"""
161
162         if Preview.parenthesize_conditional_expressions in self.mode:
163             already_parenthesized = (
164                 node.prev_sibling and node.prev_sibling.type == token.LPAR
165             )
166
167             if not already_parenthesized:
168                 lpar = Leaf(token.LPAR, "")
169                 rpar = Leaf(token.RPAR, "")
170                 node.insert_child(0, lpar)
171                 node.append_child(rpar)
172
173         yield from self.visit_default(node)
174
175     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
176         """Increase indentation level, maybe yield a line."""
177         # In blib2to3 INDENT never holds comments.
178         yield from self.line(+1)
179         yield from self.visit_default(node)
180
181     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
182         """Decrease indentation level, maybe yield a line."""
183         # The current line might still wait for trailing comments.  At DEDENT time
184         # there won't be any (they would be prefixes on the preceding NEWLINE).
185         # Emit the line then.
186         yield from self.line()
187
188         # While DEDENT has no value, its prefix may contain standalone comments
189         # that belong to the current indentation level.  Get 'em.
190         yield from self.visit_default(node)
191
192         # Finally, emit the dedent.
193         yield from self.line(-1)
194
195     def visit_stmt(
196         self, node: Node, keywords: Set[str], parens: Set[str]
197     ) -> Iterator[Line]:
198         """Visit a statement.
199
200         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
201         `def`, `with`, `class`, `assert`, and assignments.
202
203         The relevant Python language `keywords` for a given statement will be
204         NAME leaves within it. This methods puts those on a separate line.
205
206         `parens` holds a set of string leaf values immediately after which
207         invisible parens should be put.
208         """
209         normalize_invisible_parens(
210             node, parens_after=parens, mode=self.mode, features=self.features
211         )
212         for child in node.children:
213             if is_name_token(child) and child.value in keywords:
214                 yield from self.line()
215
216             yield from self.visit(child)
217
218     def visit_typeparams(self, node: Node) -> Iterator[Line]:
219         yield from self.visit_default(node)
220         node.children[0].prefix = ""
221
222     def visit_typevartuple(self, node: Node) -> Iterator[Line]:
223         yield from self.visit_default(node)
224         node.children[1].prefix = ""
225
226     def visit_paramspec(self, node: Node) -> Iterator[Line]:
227         yield from self.visit_default(node)
228         node.children[1].prefix = ""
229
230     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
231         if Preview.wrap_long_dict_values_in_parens in self.mode:
232             for i, child in enumerate(node.children):
233                 if i == 0:
234                     continue
235                 if node.children[i - 1].type == token.COLON:
236                     if child.type == syms.atom and child.children[0].type == token.LPAR:
237                         if maybe_make_parens_invisible_in_atom(
238                             child,
239                             parent=node,
240                             remove_brackets_around_comma=False,
241                         ):
242                             wrap_in_parentheses(node, child, visible=False)
243                     else:
244                         wrap_in_parentheses(node, child, visible=False)
245         yield from self.visit_default(node)
246
247     def visit_funcdef(self, node: Node) -> Iterator[Line]:
248         """Visit function definition."""
249         yield from self.line()
250
251         # Remove redundant brackets around return type annotation.
252         is_return_annotation = False
253         for child in node.children:
254             if child.type == token.RARROW:
255                 is_return_annotation = True
256             elif is_return_annotation:
257                 if child.type == syms.atom and child.children[0].type == token.LPAR:
258                     if maybe_make_parens_invisible_in_atom(
259                         child,
260                         parent=node,
261                         remove_brackets_around_comma=False,
262                     ):
263                         wrap_in_parentheses(node, child, visible=False)
264                 else:
265                     wrap_in_parentheses(node, child, visible=False)
266                 is_return_annotation = False
267
268         for child in node.children:
269             yield from self.visit(child)
270
271     def visit_match_case(self, node: Node) -> Iterator[Line]:
272         """Visit either a match or case statement."""
273         normalize_invisible_parens(
274             node, parens_after=set(), mode=self.mode, features=self.features
275         )
276
277         yield from self.line()
278         for child in node.children:
279             yield from self.visit(child)
280
281     def visit_suite(self, node: Node) -> Iterator[Line]:
282         """Visit a suite."""
283         if self.mode.is_pyi and is_stub_suite(node):
284             yield from self.visit(node.children[2])
285         else:
286             yield from self.visit_default(node)
287
288     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
289         """Visit a statement without nested statements."""
290         prev_type: Optional[int] = None
291         for child in node.children:
292             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
293                 wrap_in_parentheses(node, child, visible=False)
294             prev_type = child.type
295
296         is_suite_like = node.parent and node.parent.type in STATEMENT
297         if is_suite_like:
298             if self.mode.is_pyi and is_stub_body(node):
299                 yield from self.visit_default(node)
300             else:
301                 yield from self.line(+1)
302                 yield from self.visit_default(node)
303                 yield from self.line(-1)
304
305         else:
306             if (
307                 not self.mode.is_pyi
308                 or not node.parent
309                 or not is_stub_suite(node.parent)
310             ):
311                 yield from self.line()
312             yield from self.visit_default(node)
313
314     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
315         """Visit `async def`, `async for`, `async with`."""
316         yield from self.line()
317
318         children = iter(node.children)
319         for child in children:
320             yield from self.visit(child)
321
322             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
323                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
324                 # line.
325                 break
326
327         internal_stmt = next(children)
328         if Preview.improved_async_statements_handling in self.mode:
329             yield from self.visit(internal_stmt)
330         else:
331             for child in internal_stmt.children:
332                 yield from self.visit(child)
333
334     def visit_decorators(self, node: Node) -> Iterator[Line]:
335         """Visit decorators."""
336         for child in node.children:
337             yield from self.line()
338             yield from self.visit(child)
339
340     def visit_power(self, node: Node) -> Iterator[Line]:
341         for idx, leaf in enumerate(node.children[:-1]):
342             next_leaf = node.children[idx + 1]
343
344             if not isinstance(leaf, Leaf):
345                 continue
346
347             value = leaf.value.lower()
348             if (
349                 leaf.type == token.NUMBER
350                 and next_leaf.type == syms.trailer
351                 # Ensure that we are in an attribute trailer
352                 and next_leaf.children[0].type == token.DOT
353                 # It shouldn't wrap hexadecimal, binary and octal literals
354                 and not value.startswith(("0x", "0b", "0o"))
355                 # It shouldn't wrap complex literals
356                 and "j" not in value
357             ):
358                 wrap_in_parentheses(node, leaf)
359
360         remove_await_parens(node)
361
362         yield from self.visit_default(node)
363
364     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
365         """Remove a semicolon and put the other statement on a separate line."""
366         yield from self.line()
367
368     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
369         """End of file. Process outstanding comments and end with a newline."""
370         yield from self.visit_default(leaf)
371         yield from self.line()
372
373     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
374         if not self.current_line.bracket_tracker.any_open_brackets():
375             yield from self.line()
376         yield from self.visit_default(leaf)
377
378     def visit_factor(self, node: Node) -> Iterator[Line]:
379         """Force parentheses between a unary op and a binary power:
380
381         -2 ** 8 -> -(2 ** 8)
382         """
383         _operator, operand = node.children
384         if (
385             operand.type == syms.power
386             and len(operand.children) == 3
387             and operand.children[1].type == token.DOUBLESTAR
388         ):
389             lpar = Leaf(token.LPAR, "(")
390             rpar = Leaf(token.RPAR, ")")
391             index = operand.remove() or 0
392             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
393         yield from self.visit_default(node)
394
395     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
396         if Preview.hex_codes_in_unicode_sequences in self.mode:
397             normalize_unicode_escape_sequences(leaf)
398
399         if is_docstring(leaf) and "\\\n" not in leaf.value:
400             # We're ignoring docstrings with backslash newline escapes because changing
401             # indentation of those changes the AST representation of the code.
402             if self.mode.string_normalization:
403                 docstring = normalize_string_prefix(leaf.value)
404                 # visit_default() does handle string normalization for us, but
405                 # since this method acts differently depending on quote style (ex.
406                 # see padding logic below), there's a possibility for unstable
407                 # formatting as visit_default() is called *after*. To avoid a
408                 # situation where this function formats a docstring differently on
409                 # the second pass, normalize it early.
410                 docstring = normalize_string_quotes(docstring)
411             else:
412                 docstring = leaf.value
413             prefix = get_string_prefix(docstring)
414             docstring = docstring[len(prefix) :]  # Remove the prefix
415             quote_char = docstring[0]
416             # A natural way to remove the outer quotes is to do:
417             #   docstring = docstring.strip(quote_char)
418             # but that breaks on """""x""" (which is '""x').
419             # So we actually need to remove the first character and the next two
420             # characters but only if they are the same as the first.
421             quote_len = 1 if docstring[1] != quote_char else 3
422             docstring = docstring[quote_len:-quote_len]
423             docstring_started_empty = not docstring
424             indent = " " * 4 * self.current_line.depth
425
426             if is_multiline_string(leaf):
427                 docstring = fix_docstring(docstring, indent)
428             else:
429                 docstring = docstring.strip()
430
431             has_trailing_backslash = False
432             if docstring:
433                 # Add some padding if the docstring starts / ends with a quote mark.
434                 if docstring[0] == quote_char:
435                     docstring = " " + docstring
436                 if docstring[-1] == quote_char:
437                     docstring += " "
438                 if docstring[-1] == "\\":
439                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
440                     if backslash_count % 2:
441                         # Odd number of tailing backslashes, add some padding to
442                         # avoid escaping the closing string quote.
443                         docstring += " "
444                         has_trailing_backslash = True
445             elif not docstring_started_empty:
446                 docstring = " "
447
448             # We could enforce triple quotes at this point.
449             quote = quote_char * quote_len
450
451             # It's invalid to put closing single-character quotes on a new line.
452             if self.mode and quote_len == 3:
453                 # We need to find the length of the last line of the docstring
454                 # to find if we can add the closing quotes to the line without
455                 # exceeding the maximum line length.
456                 # If docstring is one line, we don't put the closing quotes on a
457                 # separate line because it looks ugly (#3320).
458                 lines = docstring.splitlines()
459                 last_line_length = len(lines[-1]) if docstring else 0
460
461                 # If adding closing quotes would cause the last line to exceed
462                 # the maximum line length then put a line break before the
463                 # closing quotes
464                 if (
465                     len(lines) > 1
466                     and last_line_length + quote_len > self.mode.line_length
467                     and len(indent) + quote_len <= self.mode.line_length
468                     and not has_trailing_backslash
469                 ):
470                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
471                 else:
472                     leaf.value = prefix + quote + docstring + quote
473             else:
474                 leaf.value = prefix + quote + docstring + quote
475
476         yield from self.visit_default(leaf)
477
478     def __post_init__(self) -> None:
479         """You are in a twisty little maze of passages."""
480         self.current_line = Line(mode=self.mode)
481
482         v = self.visit_stmt
483         Ø: Set[str] = set()
484         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
485         self.visit_if_stmt = partial(
486             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
487         )
488         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
489         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
490         self.visit_try_stmt = partial(
491             v, keywords={"try", "except", "else", "finally"}, parens=Ø
492         )
493         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
494         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
495         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
496         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
497         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
498         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
499         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
500         self.visit_async_funcdef = self.visit_async_stmt
501         self.visit_decorated = self.visit_decorators
502
503         # PEP 634
504         self.visit_match_stmt = self.visit_match_case
505         self.visit_case_block = self.visit_match_case
506
507
508 def transform_line(
509     line: Line, mode: Mode, features: Collection[Feature] = ()
510 ) -> Iterator[Line]:
511     """Transform a `line`, potentially splitting it into many lines.
512
513     They should fit in the allotted `line_length` but might not be able to.
514
515     `features` are syntactical features that may be used in the output.
516     """
517     if line.is_comment:
518         yield line
519         return
520
521     line_str = line_to_string(line)
522
523     ll = mode.line_length
524     sn = mode.string_normalization
525     string_merge = StringMerger(ll, sn)
526     string_paren_strip = StringParenStripper(ll, sn)
527     string_split = StringSplitter(ll, sn)
528     string_paren_wrap = StringParenWrapper(ll, sn)
529
530     transformers: List[Transformer]
531     if (
532         not line.contains_uncollapsable_type_comments()
533         and not line.should_split_rhs
534         and not line.magic_trailing_comma
535         and (
536             is_line_short_enough(line, mode=mode, line_str=line_str)
537             or line.contains_unsplittable_type_ignore()
538         )
539         and not (line.inside_brackets and line.contains_standalone_comments())
540     ):
541         # Only apply basic string preprocessing, since lines shouldn't be split here.
542         if Preview.string_processing in mode:
543             transformers = [string_merge, string_paren_strip]
544         else:
545             transformers = []
546     elif line.is_def:
547         transformers = [left_hand_split]
548     else:
549
550         def _rhs(
551             self: object, line: Line, features: Collection[Feature], mode: Mode
552         ) -> Iterator[Line]:
553             """Wraps calls to `right_hand_split`.
554
555             The calls increasingly `omit` right-hand trailers (bracket pairs with
556             content), meaning the trailers get glued together to split on another
557             bracket pair instead.
558             """
559             for omit in generate_trailers_to_omit(line, mode.line_length):
560                 lines = list(right_hand_split(line, mode, features, omit=omit))
561                 # Note: this check is only able to figure out if the first line of the
562                 # *current* transformation fits in the line length.  This is true only
563                 # for simple cases.  All others require running more transforms via
564                 # `transform_line()`.  This check doesn't know if those would succeed.
565                 if is_line_short_enough(lines[0], mode=mode):
566                     yield from lines
567                     return
568
569             # All splits failed, best effort split with no omits.
570             # This mostly happens to multiline strings that are by definition
571             # reported as not fitting a single line, as well as lines that contain
572             # trailing commas (those have to be exploded).
573             yield from right_hand_split(line, mode, features=features)
574
575         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
576         # __name__ attribute which is needed in `run_transformer` further down.
577         # Unfortunately a nested class breaks mypyc too. So a class must be created
578         # via type ... https://github.com/mypyc/mypyc/issues/884
579         rhs = type("rhs", (), {"__call__": _rhs})()
580
581         if Preview.string_processing in mode:
582             if line.inside_brackets:
583                 transformers = [
584                     string_merge,
585                     string_paren_strip,
586                     string_split,
587                     delimiter_split,
588                     standalone_comment_split,
589                     string_paren_wrap,
590                     rhs,
591                 ]
592             else:
593                 transformers = [
594                     string_merge,
595                     string_paren_strip,
596                     string_split,
597                     string_paren_wrap,
598                     rhs,
599                 ]
600         else:
601             if line.inside_brackets:
602                 transformers = [delimiter_split, standalone_comment_split, rhs]
603             else:
604                 transformers = [rhs]
605     # It's always safe to attempt hugging of power operations and pretty much every line
606     # could match.
607     transformers.append(hug_power_op)
608
609     for transform in transformers:
610         # We are accumulating lines in `result` because we might want to abort
611         # mission and return the original line in the end, or attempt a different
612         # split altogether.
613         try:
614             result = run_transformer(line, transform, mode, features, line_str=line_str)
615         except CannotTransform:
616             continue
617         else:
618             yield from result
619             break
620
621     else:
622         yield line
623
624
625 class _BracketSplitComponent(Enum):
626     head = auto()
627     body = auto()
628     tail = auto()
629
630
631 def left_hand_split(
632     line: Line, _features: Collection[Feature], mode: Mode
633 ) -> Iterator[Line]:
634     """Split line into many lines, starting with the first matching bracket pair.
635
636     Note: this usually looks weird, only use this for function definitions.
637     Prefer RHS otherwise.  This is why this function is not symmetrical with
638     :func:`right_hand_split` which also handles optional parentheses.
639     """
640     tail_leaves: List[Leaf] = []
641     body_leaves: List[Leaf] = []
642     head_leaves: List[Leaf] = []
643     current_leaves = head_leaves
644     matching_bracket: Optional[Leaf] = None
645     for leaf in line.leaves:
646         if (
647             current_leaves is body_leaves
648             and leaf.type in CLOSING_BRACKETS
649             and leaf.opening_bracket is matching_bracket
650             and isinstance(matching_bracket, Leaf)
651         ):
652             ensure_visible(leaf)
653             ensure_visible(matching_bracket)
654             current_leaves = tail_leaves if body_leaves else head_leaves
655         current_leaves.append(leaf)
656         if current_leaves is head_leaves:
657             if leaf.type in OPENING_BRACKETS:
658                 matching_bracket = leaf
659                 current_leaves = body_leaves
660     if not matching_bracket:
661         raise CannotSplit("No brackets found")
662
663     head = bracket_split_build_line(
664         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
665     )
666     body = bracket_split_build_line(
667         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
668     )
669     tail = bracket_split_build_line(
670         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
671     )
672     bracket_split_succeeded_or_raise(head, body, tail)
673     for result in (head, body, tail):
674         if result:
675             yield result
676
677
678 def right_hand_split(
679     line: Line,
680     mode: Mode,
681     features: Collection[Feature] = (),
682     omit: Collection[LeafID] = (),
683 ) -> Iterator[Line]:
684     """Split line into many lines, starting with the last matching bracket pair.
685
686     If the split was by optional parentheses, attempt splitting without them, too.
687     `omit` is a collection of closing bracket IDs that shouldn't be considered for
688     this split.
689
690     Note: running this function modifies `bracket_depth` on the leaves of `line`.
691     """
692     rhs_result = _first_right_hand_split(line, omit=omit)
693     yield from _maybe_split_omitting_optional_parens(
694         rhs_result, line, mode, features=features, omit=omit
695     )
696
697
698 def _first_right_hand_split(
699     line: Line,
700     omit: Collection[LeafID] = (),
701 ) -> RHSResult:
702     """Split the line into head, body, tail starting with the last bracket pair.
703
704     Note: this function should not have side effects. It's relied upon by
705     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
706     splitting on the right side of an assignment statement.
707     """
708     tail_leaves: List[Leaf] = []
709     body_leaves: List[Leaf] = []
710     head_leaves: List[Leaf] = []
711     current_leaves = tail_leaves
712     opening_bracket: Optional[Leaf] = None
713     closing_bracket: Optional[Leaf] = None
714     for leaf in reversed(line.leaves):
715         if current_leaves is body_leaves:
716             if leaf is opening_bracket:
717                 current_leaves = head_leaves if body_leaves else tail_leaves
718         current_leaves.append(leaf)
719         if current_leaves is tail_leaves:
720             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
721                 opening_bracket = leaf.opening_bracket
722                 closing_bracket = leaf
723                 current_leaves = body_leaves
724     if not (opening_bracket and closing_bracket and head_leaves):
725         # If there is no opening or closing_bracket that means the split failed and
726         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
727         # the matching `opening_bracket` wasn't available on `line` anymore.
728         raise CannotSplit("No brackets found")
729
730     tail_leaves.reverse()
731     body_leaves.reverse()
732     head_leaves.reverse()
733     head = bracket_split_build_line(
734         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
735     )
736     body = bracket_split_build_line(
737         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
738     )
739     tail = bracket_split_build_line(
740         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
741     )
742     bracket_split_succeeded_or_raise(head, body, tail)
743     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
744
745
746 def _maybe_split_omitting_optional_parens(
747     rhs: RHSResult,
748     line: Line,
749     mode: Mode,
750     features: Collection[Feature] = (),
751     omit: Collection[LeafID] = (),
752 ) -> Iterator[Line]:
753     if (
754         Feature.FORCE_OPTIONAL_PARENTHESES not in features
755         # the opening bracket is an optional paren
756         and rhs.opening_bracket.type == token.LPAR
757         and not rhs.opening_bracket.value
758         # the closing bracket is an optional paren
759         and rhs.closing_bracket.type == token.RPAR
760         and not rhs.closing_bracket.value
761         # it's not an import (optional parens are the only thing we can split on
762         # in this case; attempting a split without them is a waste of time)
763         and not line.is_import
764         # there are no standalone comments in the body
765         and not rhs.body.contains_standalone_comments(0)
766         # and we can actually remove the parens
767         and can_omit_invisible_parens(rhs, mode.line_length)
768     ):
769         omit = {id(rhs.closing_bracket), *omit}
770         try:
771             # The RHSResult Omitting Optional Parens.
772             rhs_oop = _first_right_hand_split(line, omit=omit)
773             if not (
774                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
775                 # the split is right after `=`
776                 and len(rhs.head.leaves) >= 2
777                 and rhs.head.leaves[-2].type == token.EQUAL
778                 # the left side of assignment contains brackets
779                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
780                 # the left side of assignment is short enough (the -1 is for the ending
781                 # optional paren)
782                 and is_line_short_enough(
783                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
784                 )
785                 # the left side of assignment won't explode further because of magic
786                 # trailing comma
787                 and rhs.head.magic_trailing_comma is None
788                 # the split by omitting optional parens isn't preferred by some other
789                 # reason
790                 and not _prefer_split_rhs_oop(rhs_oop, mode)
791             ):
792                 yield from _maybe_split_omitting_optional_parens(
793                     rhs_oop, line, mode, features=features, omit=omit
794                 )
795                 return
796
797         except CannotSplit as e:
798             if not (
799                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
800             ):
801                 raise CannotSplit(
802                     "Splitting failed, body is still too long and can't be split."
803                 ) from e
804
805             elif (
806                 rhs.head.contains_multiline_strings()
807                 or rhs.tail.contains_multiline_strings()
808             ):
809                 raise CannotSplit(
810                     "The current optional pair of parentheses is bound to fail to"
811                     " satisfy the splitting algorithm because the head or the tail"
812                     " contains multiline strings which by definition never fit one"
813                     " line."
814                 ) from e
815
816     ensure_visible(rhs.opening_bracket)
817     ensure_visible(rhs.closing_bracket)
818     for result in (rhs.head, rhs.body, rhs.tail):
819         if result:
820             yield result
821
822
823 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
824     """
825     Returns whether we should prefer the result from a split omitting optional parens.
826     """
827     has_closing_bracket_after_assign = False
828     for leaf in reversed(rhs_oop.head.leaves):
829         if leaf.type == token.EQUAL:
830             break
831         if leaf.type in CLOSING_BRACKETS:
832             has_closing_bracket_after_assign = True
833             break
834     return (
835         # contains matching brackets after the `=` (done by checking there is a
836         # closing bracket)
837         has_closing_bracket_after_assign
838         or (
839             # the split is actually from inside the optional parens (done by checking
840             # the first line still contains the `=`)
841             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
842             # the first line is short enough
843             and is_line_short_enough(rhs_oop.head, mode=mode)
844         )
845         # contains unsplittable type ignore
846         or rhs_oop.head.contains_unsplittable_type_ignore()
847         or rhs_oop.body.contains_unsplittable_type_ignore()
848         or rhs_oop.tail.contains_unsplittable_type_ignore()
849     )
850
851
852 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
853     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
854
855     Do nothing otherwise.
856
857     A left- or right-hand split is based on a pair of brackets. Content before
858     (and including) the opening bracket is left on one line, content inside the
859     brackets is put on a separate line, and finally content starting with and
860     following the closing bracket is put on a separate line.
861
862     Those are called `head`, `body`, and `tail`, respectively. If the split
863     produced the same line (all content in `head`) or ended up with an empty `body`
864     and the `tail` is just the closing bracket, then it's considered failed.
865     """
866     tail_len = len(str(tail).strip())
867     if not body:
868         if tail_len == 0:
869             raise CannotSplit("Splitting brackets produced the same line")
870
871         elif tail_len < 3:
872             raise CannotSplit(
873                 f"Splitting brackets on an empty body to save {tail_len} characters is"
874                 " not worth it"
875             )
876
877
878 def bracket_split_build_line(
879     leaves: List[Leaf],
880     original: Line,
881     opening_bracket: Leaf,
882     *,
883     component: _BracketSplitComponent,
884 ) -> Line:
885     """Return a new line with given `leaves` and respective comments from `original`.
886
887     If it's the head component, brackets will be tracked so trailing commas are
888     respected.
889
890     If it's the body component, the result line is one-indented inside brackets and as
891     such has its first leaf's prefix normalized and a trailing comma added when
892     expected.
893     """
894     result = Line(mode=original.mode, depth=original.depth)
895     if component is _BracketSplitComponent.body:
896         result.inside_brackets = True
897         result.depth += 1
898         if leaves:
899             # Since body is a new indent level, remove spurious leading whitespace.
900             normalize_prefix(leaves[0], inside_brackets=True)
901             # Ensure a trailing comma for imports and standalone function arguments, but
902             # be careful not to add one after any comments or within type annotations.
903             no_commas = (
904                 original.is_def
905                 and opening_bracket.value == "("
906                 and not any(leaf.type == token.COMMA for leaf in leaves)
907                 # In particular, don't add one within a parenthesized return annotation.
908                 # Unfortunately the indicator we're in a return annotation (RARROW) may
909                 # be defined directly in the parent node, the parent of the parent ...
910                 # and so on depending on how complex the return annotation is.
911                 # This isn't perfect and there's some false negatives but they are in
912                 # contexts were a comma is actually fine.
913                 and not any(
914                     node.prev_sibling.type == RARROW
915                     for node in (
916                         leaves[0].parent,
917                         getattr(leaves[0].parent, "parent", None),
918                     )
919                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
920                 )
921             )
922
923             if original.is_import or no_commas:
924                 for i in range(len(leaves) - 1, -1, -1):
925                     if leaves[i].type == STANDALONE_COMMENT:
926                         continue
927
928                     if leaves[i].type != token.COMMA:
929                         new_comma = Leaf(token.COMMA, ",")
930                         leaves.insert(i + 1, new_comma)
931                     break
932
933     leaves_to_track: Set[LeafID] = set()
934     if component is _BracketSplitComponent.head:
935         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
936     # Populate the line
937     for leaf in leaves:
938         result.append(
939             leaf,
940             preformatted=True,
941             track_bracket=id(leaf) in leaves_to_track,
942         )
943         for comment_after in original.comments_after(leaf):
944             result.append(comment_after, preformatted=True)
945     if component is _BracketSplitComponent.body and should_split_line(
946         result, opening_bracket
947     ):
948         result.should_split_rhs = True
949     return result
950
951
952 def dont_increase_indentation(split_func: Transformer) -> Transformer:
953     """Normalize prefix of the first leaf in every line returned by `split_func`.
954
955     This is a decorator over relevant split functions.
956     """
957
958     @wraps(split_func)
959     def split_wrapper(
960         line: Line, features: Collection[Feature], mode: Mode
961     ) -> Iterator[Line]:
962         for split_line in split_func(line, features, mode):
963             normalize_prefix(split_line.leaves[0], inside_brackets=True)
964             yield split_line
965
966     return split_wrapper
967
968
969 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
970     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
971         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
972             return leaf_idx
973     return None
974
975
976 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
977     if (
978         safe
979         and delimiter_priority == COMMA_PRIORITY
980         and line.leaves[-1].type != token.COMMA
981         and line.leaves[-1].type != STANDALONE_COMMENT
982     ):
983         new_comma = Leaf(token.COMMA, ",")
984         line.append(new_comma)
985     return line
986
987
988 @dont_increase_indentation
989 def delimiter_split(
990     line: Line, features: Collection[Feature], mode: Mode
991 ) -> Iterator[Line]:
992     """Split according to delimiters of the highest priority.
993
994     If the appropriate Features are given, the split will add trailing commas
995     also in function signatures and calls that contain `*` and `**`.
996     """
997     try:
998         last_leaf = line.leaves[-1]
999     except IndexError:
1000         raise CannotSplit("Line empty") from None
1001
1002     bt = line.bracket_tracker
1003     try:
1004         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1005     except ValueError:
1006         raise CannotSplit("No delimiters found") from None
1007
1008     if delimiter_priority == DOT_PRIORITY:
1009         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
1010             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1011
1012     current_line = Line(
1013         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1014     )
1015     lowest_depth = sys.maxsize
1016     trailing_comma_safe = True
1017
1018     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1019         """Append `leaf` to current line or to new line if appending impossible."""
1020         nonlocal current_line
1021         try:
1022             current_line.append_safe(leaf, preformatted=True)
1023         except ValueError:
1024             yield current_line
1025
1026             current_line = Line(
1027                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1028             )
1029             current_line.append(leaf)
1030
1031     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1032     for leaf_idx, leaf in enumerate(line.leaves):
1033         yield from append_to_line(leaf)
1034
1035         for comment_after in line.comments_after(leaf):
1036             yield from append_to_line(comment_after)
1037
1038         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1039         if leaf.bracket_depth == lowest_depth:
1040             if is_vararg(leaf, within={syms.typedargslist}):
1041                 trailing_comma_safe = (
1042                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1043                 )
1044             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1045                 trailing_comma_safe = (
1046                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1047                 )
1048
1049         if (
1050             Preview.add_trailing_comma_consistently in mode
1051             and last_leaf.type == STANDALONE_COMMENT
1052             and leaf_idx == last_non_comment_leaf
1053         ):
1054             current_line = _safe_add_trailing_comma(
1055                 trailing_comma_safe, delimiter_priority, current_line
1056             )
1057
1058         leaf_priority = bt.delimiters.get(id(leaf))
1059         if leaf_priority == delimiter_priority:
1060             yield current_line
1061
1062             current_line = Line(
1063                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1064             )
1065     if current_line:
1066         current_line = _safe_add_trailing_comma(
1067             trailing_comma_safe, delimiter_priority, current_line
1068         )
1069         yield current_line
1070
1071
1072 @dont_increase_indentation
1073 def standalone_comment_split(
1074     line: Line, features: Collection[Feature], mode: Mode
1075 ) -> Iterator[Line]:
1076     """Split standalone comments from the rest of the line."""
1077     if not line.contains_standalone_comments(0):
1078         raise CannotSplit("Line does not have any standalone comments")
1079
1080     current_line = Line(
1081         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1082     )
1083
1084     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1085         """Append `leaf` to current line or to new line if appending impossible."""
1086         nonlocal current_line
1087         try:
1088             current_line.append_safe(leaf, preformatted=True)
1089         except ValueError:
1090             yield current_line
1091
1092             current_line = Line(
1093                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1094             )
1095             current_line.append(leaf)
1096
1097     for leaf in line.leaves:
1098         yield from append_to_line(leaf)
1099
1100         for comment_after in line.comments_after(leaf):
1101             yield from append_to_line(comment_after)
1102
1103     if current_line:
1104         yield current_line
1105
1106
1107 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1108     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1109     else.
1110
1111     Note: don't use backslashes for formatting or you'll lose your voting rights.
1112     """
1113     if not inside_brackets:
1114         spl = leaf.prefix.split("#")
1115         if "\\" not in spl[0]:
1116             nl_count = spl[-1].count("\n")
1117             if len(spl) > 1:
1118                 nl_count -= 1
1119             leaf.prefix = "\n" * nl_count
1120             return
1121
1122     leaf.prefix = ""
1123
1124
1125 def normalize_invisible_parens(
1126     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1127 ) -> None:
1128     """Make existing optional parentheses invisible or create new ones.
1129
1130     `parens_after` is a set of string leaf values immediately after which parens
1131     should be put.
1132
1133     Standardizes on visible parentheses for single-element tuples, and keeps
1134     existing visible parentheses for other tuples and generator expressions.
1135     """
1136     for pc in list_comments(node.prefix, is_endmarker=False):
1137         if pc.value in FMT_OFF:
1138             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1139             return
1140
1141     # The multiple context managers grammar has a different pattern, thus this is
1142     # separate from the for-loop below. This possibly wraps them in invisible parens,
1143     # and later will be removed in remove_with_parens when needed.
1144     if node.type == syms.with_stmt:
1145         _maybe_wrap_cms_in_parens(node, mode, features)
1146
1147     check_lpar = False
1148     for index, child in enumerate(list(node.children)):
1149         # Fixes a bug where invisible parens are not properly stripped from
1150         # assignment statements that contain type annotations.
1151         if isinstance(child, Node) and child.type == syms.annassign:
1152             normalize_invisible_parens(
1153                 child, parens_after=parens_after, mode=mode, features=features
1154             )
1155
1156         # Add parentheses around long tuple unpacking in assignments.
1157         if (
1158             index == 0
1159             and isinstance(child, Node)
1160             and child.type == syms.testlist_star_expr
1161         ):
1162             check_lpar = True
1163
1164         if check_lpar:
1165             if (
1166                 child.type == syms.atom
1167                 and node.type == syms.for_stmt
1168                 and isinstance(child.prev_sibling, Leaf)
1169                 and child.prev_sibling.type == token.NAME
1170                 and child.prev_sibling.value == "for"
1171             ):
1172                 if maybe_make_parens_invisible_in_atom(
1173                     child,
1174                     parent=node,
1175                     remove_brackets_around_comma=True,
1176                 ):
1177                     wrap_in_parentheses(node, child, visible=False)
1178             elif isinstance(child, Node) and node.type == syms.with_stmt:
1179                 remove_with_parens(child, node)
1180             elif child.type == syms.atom:
1181                 if maybe_make_parens_invisible_in_atom(
1182                     child,
1183                     parent=node,
1184                 ):
1185                     wrap_in_parentheses(node, child, visible=False)
1186             elif is_one_tuple(child):
1187                 wrap_in_parentheses(node, child, visible=True)
1188             elif node.type == syms.import_from:
1189                 _normalize_import_from(node, child, index)
1190                 break
1191             elif (
1192                 index == 1
1193                 and child.type == token.STAR
1194                 and node.type == syms.except_clause
1195             ):
1196                 # In except* (PEP 654), the star is actually part of
1197                 # of the keyword. So we need to skip the insertion of
1198                 # invisible parentheses to work more precisely.
1199                 continue
1200
1201             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1202                 wrap_in_parentheses(node, child, visible=False)
1203
1204         comma_check = child.type == token.COMMA
1205
1206         check_lpar = isinstance(child, Leaf) and (
1207             child.value in parens_after or comma_check
1208         )
1209
1210
1211 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1212     # "import from" nodes store parentheses directly as part of
1213     # the statement
1214     if is_lpar_token(child):
1215         assert is_rpar_token(parent.children[-1])
1216         # make parentheses invisible
1217         child.value = ""
1218         parent.children[-1].value = ""
1219     elif child.type != token.STAR:
1220         # insert invisible parentheses
1221         parent.insert_child(index, Leaf(token.LPAR, ""))
1222         parent.append_child(Leaf(token.RPAR, ""))
1223
1224
1225 def remove_await_parens(node: Node) -> None:
1226     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1227         if (
1228             node.children[1].type == syms.atom
1229             and node.children[1].children[0].type == token.LPAR
1230         ):
1231             if maybe_make_parens_invisible_in_atom(
1232                 node.children[1],
1233                 parent=node,
1234                 remove_brackets_around_comma=True,
1235             ):
1236                 wrap_in_parentheses(node, node.children[1], visible=False)
1237
1238             # Since await is an expression we shouldn't remove
1239             # brackets in cases where this would change
1240             # the AST due to operator precedence.
1241             # Therefore we only aim to remove brackets around
1242             # power nodes that aren't also await expressions themselves.
1243             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1244             # N.B. We've still removed any redundant nested brackets though :)
1245             opening_bracket = cast(Leaf, node.children[1].children[0])
1246             closing_bracket = cast(Leaf, node.children[1].children[-1])
1247             bracket_contents = node.children[1].children[1]
1248             if isinstance(bracket_contents, Node):
1249                 if bracket_contents.type != syms.power:
1250                     ensure_visible(opening_bracket)
1251                     ensure_visible(closing_bracket)
1252                 elif (
1253                     bracket_contents.type == syms.power
1254                     and bracket_contents.children[0].type == token.AWAIT
1255                 ):
1256                     ensure_visible(opening_bracket)
1257                     ensure_visible(closing_bracket)
1258                     # If we are in a nested await then recurse down.
1259                     remove_await_parens(bracket_contents)
1260
1261
1262 def _maybe_wrap_cms_in_parens(
1263     node: Node, mode: Mode, features: Collection[Feature]
1264 ) -> None:
1265     """When enabled and safe, wrap the multiple context managers in invisible parens.
1266
1267     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1268     """
1269     if (
1270         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1271         or Preview.wrap_multiple_context_managers_in_parens not in mode
1272         or len(node.children) <= 2
1273         # If it's an atom, it's already wrapped in parens.
1274         or node.children[1].type == syms.atom
1275     ):
1276         return
1277     colon_index: Optional[int] = None
1278     for i in range(2, len(node.children)):
1279         if node.children[i].type == token.COLON:
1280             colon_index = i
1281             break
1282     if colon_index is not None:
1283         lpar = Leaf(token.LPAR, "")
1284         rpar = Leaf(token.RPAR, "")
1285         context_managers = node.children[1:colon_index]
1286         for child in context_managers:
1287             child.remove()
1288         # After wrapping, the with_stmt will look like this:
1289         #   with_stmt
1290         #     NAME 'with'
1291         #     atom
1292         #       LPAR ''
1293         #       testlist_gexp
1294         #         ... <-- context_managers
1295         #       /testlist_gexp
1296         #       RPAR ''
1297         #     /atom
1298         #     COLON ':'
1299         new_child = Node(
1300             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1301         )
1302         node.insert_child(1, new_child)
1303
1304
1305 def remove_with_parens(node: Node, parent: Node) -> None:
1306     """Recursively hide optional parens in `with` statements."""
1307     # Removing all unnecessary parentheses in with statements in one pass is a tad
1308     # complex as different variations of bracketed statements result in pretty
1309     # different parse trees:
1310     #
1311     # with (open("file")) as f:                       # this is an asexpr_test
1312     #     ...
1313     #
1314     # with (open("file") as f):                       # this is an atom containing an
1315     #     ...                                         # asexpr_test
1316     #
1317     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1318     #     ...                                         # asexpr_test
1319     #
1320     # with (open("file") as f, open("file") as f):    # an atom containing a
1321     #     ...                                         # testlist_gexp which then
1322     #                                                 # contains multiple asexpr_test(s)
1323     if node.type == syms.atom:
1324         if maybe_make_parens_invisible_in_atom(
1325             node,
1326             parent=parent,
1327             remove_brackets_around_comma=True,
1328         ):
1329             wrap_in_parentheses(parent, node, visible=False)
1330         if isinstance(node.children[1], Node):
1331             remove_with_parens(node.children[1], node)
1332     elif node.type == syms.testlist_gexp:
1333         for child in node.children:
1334             if isinstance(child, Node):
1335                 remove_with_parens(child, node)
1336     elif node.type == syms.asexpr_test and not any(
1337         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1338     ):
1339         if maybe_make_parens_invisible_in_atom(
1340             node.children[0],
1341             parent=node,
1342             remove_brackets_around_comma=True,
1343         ):
1344             wrap_in_parentheses(node, node.children[0], visible=False)
1345
1346
1347 def maybe_make_parens_invisible_in_atom(
1348     node: LN,
1349     parent: LN,
1350     remove_brackets_around_comma: bool = False,
1351 ) -> bool:
1352     """If it's safe, make the parens in the atom `node` invisible, recursively.
1353     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1354     as they are redundant.
1355
1356     Returns whether the node should itself be wrapped in invisible parentheses.
1357     """
1358     if (
1359         node.type != syms.atom
1360         or is_empty_tuple(node)
1361         or is_one_tuple(node)
1362         or (is_yield(node) and parent.type != syms.expr_stmt)
1363         or (
1364             # This condition tries to prevent removing non-optional brackets
1365             # around a tuple, however, can be a bit overzealous so we provide
1366             # and option to skip this check for `for` and `with` statements.
1367             not remove_brackets_around_comma
1368             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1369         )
1370         or is_tuple_containing_walrus(node)
1371     ):
1372         return False
1373
1374     if is_walrus_assignment(node):
1375         if parent.type in [
1376             syms.annassign,
1377             syms.expr_stmt,
1378             syms.assert_stmt,
1379             syms.return_stmt,
1380             syms.except_clause,
1381             syms.funcdef,
1382             syms.with_stmt,
1383             # these ones aren't useful to end users, but they do please fuzzers
1384             syms.for_stmt,
1385             syms.del_stmt,
1386             syms.for_stmt,
1387         ]:
1388             return False
1389
1390     first = node.children[0]
1391     last = node.children[-1]
1392     if is_lpar_token(first) and is_rpar_token(last):
1393         middle = node.children[1]
1394         # make parentheses invisible
1395         first.value = ""
1396         last.value = ""
1397         maybe_make_parens_invisible_in_atom(
1398             middle,
1399             parent=parent,
1400             remove_brackets_around_comma=remove_brackets_around_comma,
1401         )
1402
1403         if is_atom_with_invisible_parens(middle):
1404             # Strip the invisible parens from `middle` by replacing
1405             # it with the child in-between the invisible parens
1406             middle.replace(middle.children[1])
1407
1408         return False
1409
1410     return True
1411
1412
1413 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1414     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1415
1416     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1417         return False
1418
1419     # We're essentially checking if the body is delimited by commas and there's more
1420     # than one of them (we're excluding the trailing comma and if the delimiter priority
1421     # is still commas, that means there's more).
1422     exclude = set()
1423     trailing_comma = False
1424     try:
1425         last_leaf = line.leaves[-1]
1426         if last_leaf.type == token.COMMA:
1427             trailing_comma = True
1428             exclude.add(id(last_leaf))
1429         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1430     except (IndexError, ValueError):
1431         return False
1432
1433     return max_priority == COMMA_PRIORITY and (
1434         (line.mode.magic_trailing_comma and trailing_comma)
1435         # always explode imports
1436         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1437     )
1438
1439
1440 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1441     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1442
1443     Brackets can be omitted if the entire trailer up to and including
1444     a preceding closing bracket fits in one line.
1445
1446     Yielded sets are cumulative (contain results of previous yields, too).  First
1447     set is empty, unless the line should explode, in which case bracket pairs until
1448     the one that needs to explode are omitted.
1449     """
1450
1451     omit: Set[LeafID] = set()
1452     if not line.magic_trailing_comma:
1453         yield omit
1454
1455     length = 4 * line.depth
1456     opening_bracket: Optional[Leaf] = None
1457     closing_bracket: Optional[Leaf] = None
1458     inner_brackets: Set[LeafID] = set()
1459     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1460         length += leaf_length
1461         if length > line_length:
1462             break
1463
1464         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1465         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1466             break
1467
1468         if opening_bracket:
1469             if leaf is opening_bracket:
1470                 opening_bracket = None
1471             elif leaf.type in CLOSING_BRACKETS:
1472                 prev = line.leaves[index - 1] if index > 0 else None
1473                 if (
1474                     prev
1475                     and prev.type == token.COMMA
1476                     and leaf.opening_bracket is not None
1477                     and not is_one_sequence_between(
1478                         leaf.opening_bracket, leaf, line.leaves
1479                     )
1480                 ):
1481                     # Never omit bracket pairs with trailing commas.
1482                     # We need to explode on those.
1483                     break
1484
1485                 inner_brackets.add(id(leaf))
1486         elif leaf.type in CLOSING_BRACKETS:
1487             prev = line.leaves[index - 1] if index > 0 else None
1488             if prev and prev.type in OPENING_BRACKETS:
1489                 # Empty brackets would fail a split so treat them as "inner"
1490                 # brackets (e.g. only add them to the `omit` set if another
1491                 # pair of brackets was good enough.
1492                 inner_brackets.add(id(leaf))
1493                 continue
1494
1495             if closing_bracket:
1496                 omit.add(id(closing_bracket))
1497                 omit.update(inner_brackets)
1498                 inner_brackets.clear()
1499                 yield omit
1500
1501             if (
1502                 prev
1503                 and prev.type == token.COMMA
1504                 and leaf.opening_bracket is not None
1505                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1506             ):
1507                 # Never omit bracket pairs with trailing commas.
1508                 # We need to explode on those.
1509                 break
1510
1511             if leaf.value:
1512                 opening_bracket = leaf.opening_bracket
1513                 closing_bracket = leaf
1514
1515
1516 def run_transformer(
1517     line: Line,
1518     transform: Transformer,
1519     mode: Mode,
1520     features: Collection[Feature],
1521     *,
1522     line_str: str = "",
1523 ) -> List[Line]:
1524     if not line_str:
1525         line_str = line_to_string(line)
1526     result: List[Line] = []
1527     for transformed_line in transform(line, features, mode):
1528         if str(transformed_line).strip("\n") == line_str:
1529             raise CannotTransform("Line transformer returned an unchanged result")
1530
1531         result.extend(transform_line(transformed_line, mode=mode, features=features))
1532
1533     features_set = set(features)
1534     if (
1535         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1536         or transform.__class__.__name__ != "rhs"
1537         or not line.bracket_tracker.invisible
1538         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1539         or line.contains_multiline_strings()
1540         or result[0].contains_uncollapsable_type_comments()
1541         or result[0].contains_unsplittable_type_ignore()
1542         or is_line_short_enough(result[0], mode=mode)
1543         # If any leaves have no parents (which _can_ occur since
1544         # `transform(line)` potentially destroys the line's underlying node
1545         # structure), then we can't proceed. Doing so would cause the below
1546         # call to `append_leaves()` to fail.
1547         or any(leaf.parent is None for leaf in line.leaves)
1548     ):
1549         return result
1550
1551     line_copy = line.clone()
1552     append_leaves(line_copy, line, line.leaves)
1553     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1554     second_opinion = run_transformer(
1555         line_copy, transform, mode, features_fop, line_str=line_str
1556     )
1557     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1558         result = second_opinion
1559     return result