src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import dataclass
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     append_leaves,
  20     can_be_split,
  21     can_omit_invisible_parens,
  22     is_line_short_enough,
  23     line_to_string,
  24 )
  25 from black.mode import Feature, Mode, Preview
  26 from black.nodes import (
  27     ASSIGNMENTS,
  28     BRACKETS,
  29     CLOSING_BRACKETS,
  30     OPENING_BRACKETS,
  31     RARROW,
  32     STANDALONE_COMMENT,
  33     STATEMENT,
  34     WHITESPACE,
  35     Visitor,
  36     ensure_visible,
  37     is_arith_like,
  38     is_atom_with_invisible_parens,
  39     is_docstring,
  40     is_empty_tuple,
  41     is_lpar_token,
  42     is_multiline_string,
  43     is_name_token,
  44     is_one_sequence_between,
  45     is_one_tuple,
  46     is_rpar_token,
  47     is_stub_body,
  48     is_stub_suite,
  49     is_vararg,
  50     is_walrus_assignment,
  51     is_yield,
  52     syms,
  53     wrap_in_parentheses,
  54 )
  55 from black.numerics import normalize_numeric_literal
  56 from black.strings import (
  57     fix_docstring,
  58     get_string_prefix,
  59     normalize_string_prefix,
  60     normalize_string_quotes,
  61 )
  62 from black.trans import (
  63     CannotTransform,
  64     StringMerger,
  65     StringParenStripper,
  66     StringParenWrapper,
  67     StringSplitter,
  68     Transformer,
  69     hug_power_op,
  70 )
  71 from blib2to3.pgen2 import token
  72 from blib2to3.pytree import Leaf, Node
  73
  74 # types
  75 LeafID = int
  76 LN = Union[Leaf, Node]
  77
  78
  79 class CannotSplit(CannotTransform):
  80     """A readable split that fits the allotted line length is impossible."""
  81
  82
  83 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  84 # See also https://github.com/mypyc/mypyc/issues/827.
  85 class LineGenerator(Visitor[Line]):
  86     """Generates reformatted Line objects.  Empty lines are not emitted.
  87
  88     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  89     in ways that will no longer stringify to valid Python code on the tree.
  90     """
  91
  92     def __init__(self, mode: Mode) -> None:
  93         self.mode = mode
  94         self.current_line: Line
  95         self.__post_init__()
  96
  97     def line(self, indent: int = 0) -> Iterator[Line]:
  98         """Generate a line.
  99
 100         If the line is empty, only emit if it makes sense.
 101         If the line is too long, split it first and then generate.
 102
 103         If any lines were generated, set up a new current_line.
 104         """
 105         if not self.current_line:
 106             self.current_line.depth += indent
 107             return  # Line is empty, don't emit. Creating a new one unnecessary.
 108
 109         complete_line = self.current_line
 110         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 111         yield complete_line
 112
 113     def visit_default(self, node: LN) -> Iterator[Line]:
 114         """Default `visit_*()` implementation. Recurses to children of `node`."""
 115         if isinstance(node, Leaf):
 116             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 117             for comment in generate_comments(node, preview=self.mode.preview):
 118                 if any_open_brackets:
 119                     # any comment within brackets is subject to splitting
 120                     self.current_line.append(comment)
 121                 elif comment.type == token.COMMENT:
 122                     # regular trailing comment
 123                     self.current_line.append(comment)
 124                     yield from self.line()
 125
 126                 else:
 127                     # regular standalone comment
 128                     yield from self.line()
 129
 130                     self.current_line.append(comment)
 131                     yield from self.line()
 132
 133             normalize_prefix(node, inside_brackets=any_open_brackets)
 134             if self.mode.string_normalization and node.type == token.STRING:
 135                 node.value = normalize_string_prefix(node.value)
 136                 node.value = normalize_string_quotes(node.value)
 137             if node.type == token.NUMBER:
 138                 normalize_numeric_literal(node)
 139             if node.type not in WHITESPACE:
 140                 self.current_line.append(node)
 141         yield from super().visit_default(node)
 142
 143     def visit_test(self, node: Node) -> Iterator[Line]:
 144         """Visit an `x if y else z` test"""
 145
 146         if Preview.parenthesize_conditional_expressions in self.mode:
 147             already_parenthesized = (
 148                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 149             )
 150
 151             if not already_parenthesized:
 152                 lpar = Leaf(token.LPAR, "")
 153                 rpar = Leaf(token.RPAR, "")
 154                 node.insert_child(0, lpar)
 155                 node.append_child(rpar)
 156
 157         yield from self.visit_default(node)
 158
 159     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 160         """Increase indentation level, maybe yield a line."""
 161         # In blib2to3 INDENT never holds comments.
 162         yield from self.line(+1)
 163         yield from self.visit_default(node)
 164
 165     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 166         """Decrease indentation level, maybe yield a line."""
 167         # The current line might still wait for trailing comments.  At DEDENT time
 168         # there won't be any (they would be prefixes on the preceding NEWLINE).
 169         # Emit the line then.
 170         yield from self.line()
 171
 172         # While DEDENT has no value, its prefix may contain standalone comments
 173         # that belong to the current indentation level.  Get 'em.
 174         yield from self.visit_default(node)
 175
 176         # Finally, emit the dedent.
 177         yield from self.line(-1)
 178
 179     def visit_stmt(
 180         self, node: Node, keywords: Set[str], parens: Set[str]
 181     ) -> Iterator[Line]:
 182         """Visit a statement.
 183
 184         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 185         `def`, `with`, `class`, `assert`, and assignments.
 186
 187         The relevant Python language `keywords` for a given statement will be
 188         NAME leaves within it. This methods puts those on a separate line.
 189
 190         `parens` holds a set of string leaf values immediately after which
 191         invisible parens should be put.
 192         """
 193         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
 194         for child in node.children:
 195             if is_name_token(child) and child.value in keywords:
 196                 yield from self.line()
 197
 198             yield from self.visit(child)
 199
 200     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 201         if Preview.wrap_long_dict_values_in_parens in self.mode:
 202             for i, child in enumerate(node.children):
 203                 if i == 0:
 204                     continue
 205                 if node.children[i - 1].type == token.COLON:
 206                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 207                         if maybe_make_parens_invisible_in_atom(
 208                             child,
 209                             parent=node,
 210                             remove_brackets_around_comma=False,
 211                         ):
 212                             wrap_in_parentheses(node, child, visible=False)
 213                     else:
 214                         wrap_in_parentheses(node, child, visible=False)
 215         yield from self.visit_default(node)
 216
 217     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 218         """Visit function definition."""
 219         if Preview.annotation_parens not in self.mode:
 220             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
 221         else:
 222             yield from self.line()
 223
 224             # Remove redundant brackets around return type annotation.
 225             is_return_annotation = False
 226             for child in node.children:
 227                 if child.type == token.RARROW:
 228                     is_return_annotation = True
 229                 elif is_return_annotation:
 230                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 231                         if maybe_make_parens_invisible_in_atom(
 232                             child,
 233                             parent=node,
 234                             remove_brackets_around_comma=False,
 235                         ):
 236                             wrap_in_parentheses(node, child, visible=False)
 237                     else:
 238                         wrap_in_parentheses(node, child, visible=False)
 239                     is_return_annotation = False
 240
 241             for child in node.children:
 242                 yield from self.visit(child)
 243
 244     def visit_match_case(self, node: Node) -> Iterator[Line]:
 245         """Visit either a match or case statement."""
 246         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
 247
 248         yield from self.line()
 249         for child in node.children:
 250             yield from self.visit(child)
 251
 252     def visit_suite(self, node: Node) -> Iterator[Line]:
 253         """Visit a suite."""
 254         if self.mode.is_pyi and is_stub_suite(node):
 255             yield from self.visit(node.children[2])
 256         else:
 257             yield from self.visit_default(node)
 258
 259     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 260         """Visit a statement without nested statements."""
 261         prev_type: Optional[int] = None
 262         for child in node.children:
 263             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 264                 wrap_in_parentheses(node, child, visible=False)
 265             prev_type = child.type
 266
 267         is_suite_like = node.parent and node.parent.type in STATEMENT
 268         if is_suite_like:
 269             if self.mode.is_pyi and is_stub_body(node):
 270                 yield from self.visit_default(node)
 271             else:
 272                 yield from self.line(+1)
 273                 yield from self.visit_default(node)
 274                 yield from self.line(-1)
 275
 276         else:
 277             if (
 278                 not self.mode.is_pyi
 279                 or not node.parent
 280                 or not is_stub_suite(node.parent)
 281             ):
 282                 yield from self.line()
 283             yield from self.visit_default(node)
 284
 285     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 286         """Visit `async def`, `async for`, `async with`."""
 287         yield from self.line()
 288
 289         children = iter(node.children)
 290         for child in children:
 291             yield from self.visit(child)
 292
 293             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 294                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 295                 # line.
 296                 break
 297
 298         internal_stmt = next(children)
 299         for child in internal_stmt.children:
 300             yield from self.visit(child)
 301
 302     def visit_decorators(self, node: Node) -> Iterator[Line]:
 303         """Visit decorators."""
 304         for child in node.children:
 305             yield from self.line()
 306             yield from self.visit(child)
 307
 308     def visit_power(self, node: Node) -> Iterator[Line]:
 309         for idx, leaf in enumerate(node.children[:-1]):
 310             next_leaf = node.children[idx + 1]
 311
 312             if not isinstance(leaf, Leaf):
 313                 continue
 314
 315             value = leaf.value.lower()
 316             if (
 317                 leaf.type == token.NUMBER
 318                 and next_leaf.type == syms.trailer
 319                 # Ensure that we are in an attribute trailer
 320                 and next_leaf.children[0].type == token.DOT
 321                 # It shouldn't wrap hexadecimal, binary and octal literals
 322                 and not value.startswith(("0x", "0b", "0o"))
 323                 # It shouldn't wrap complex literals
 324                 and "j" not in value
 325             ):
 326                 wrap_in_parentheses(node, leaf)
 327
 328         if Preview.remove_redundant_parens in self.mode:
 329             remove_await_parens(node)
 330
 331         yield from self.visit_default(node)
 332
 333     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 334         """Remove a semicolon and put the other statement on a separate line."""
 335         yield from self.line()
 336
 337     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 338         """End of file. Process outstanding comments and end with a newline."""
 339         yield from self.visit_default(leaf)
 340         yield from self.line()
 341
 342     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 343         if not self.current_line.bracket_tracker.any_open_brackets():
 344             yield from self.line()
 345         yield from self.visit_default(leaf)
 346
 347     def visit_factor(self, node: Node) -> Iterator[Line]:
 348         """Force parentheses between a unary op and a binary power:
 349
 350         -2 ** 8 -> -(2 ** 8)
 351         """
 352         _operator, operand = node.children
 353         if (
 354             operand.type == syms.power
 355             and len(operand.children) == 3
 356             and operand.children[1].type == token.DOUBLESTAR
 357         ):
 358             lpar = Leaf(token.LPAR, "(")
 359             rpar = Leaf(token.RPAR, ")")
 360             index = operand.remove() or 0
 361             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 362         yield from self.visit_default(node)
 363
 364     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 365         if is_docstring(leaf) and "\\\n" not in leaf.value:
 366             # We're ignoring docstrings with backslash newline escapes because changing
 367             # indentation of those changes the AST representation of the code.
 368             if Preview.normalize_docstring_quotes_and_prefixes_properly in self.mode:
 369                 # There was a bug where --skip-string-normalization wouldn't stop us
 370                 # from normalizing docstring prefixes. To maintain stability, we can
 371                 # only address this buggy behaviour while the preview style is enabled.
 372                 if self.mode.string_normalization:
 373                     docstring = normalize_string_prefix(leaf.value)
 374                     # visit_default() does handle string normalization for us, but
 375                     # since this method acts differently depending on quote style (ex.
 376                     # see padding logic below), there's a possibility for unstable
 377                     # formatting as visit_default() is called *after*. To avoid a
 378                     # situation where this function formats a docstring differently on
 379                     # the second pass, normalize it early.
 380                     docstring = normalize_string_quotes(docstring)
 381                 else:
 382                     docstring = leaf.value
 383             else:
 384                 # ... otherwise, we'll keep the buggy behaviour >.<
 385                 docstring = normalize_string_prefix(leaf.value)
 386             prefix = get_string_prefix(docstring)
 387             docstring = docstring[len(prefix) :]  # Remove the prefix
 388             quote_char = docstring[0]
 389             # A natural way to remove the outer quotes is to do:
 390             #   docstring = docstring.strip(quote_char)
 391             # but that breaks on """""x""" (which is '""x').
 392             # So we actually need to remove the first character and the next two
 393             # characters but only if they are the same as the first.
 394             quote_len = 1 if docstring[1] != quote_char else 3
 395             docstring = docstring[quote_len:-quote_len]
 396             docstring_started_empty = not docstring
 397             indent = " " * 4 * self.current_line.depth
 398
 399             if is_multiline_string(leaf):
 400                 docstring = fix_docstring(docstring, indent)
 401             else:
 402                 docstring = docstring.strip()
 403
 404             has_trailing_backslash = False
 405             if docstring:
 406                 # Add some padding if the docstring starts / ends with a quote mark.
 407                 if docstring[0] == quote_char:
 408                     docstring = " " + docstring
 409                 if docstring[-1] == quote_char:
 410                     docstring += " "
 411                 if docstring[-1] == "\\":
 412                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 413                     if backslash_count % 2:
 414                         # Odd number of tailing backslashes, add some padding to
 415                         # avoid escaping the closing string quote.
 416                         docstring += " "
 417                         has_trailing_backslash = True
 418             elif not docstring_started_empty:
 419                 docstring = " "
 420
 421             # We could enforce triple quotes at this point.
 422             quote = quote_char * quote_len
 423
 424             # It's invalid to put closing single-character quotes on a new line.
 425             if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
 426                 # We need to find the length of the last line of the docstring
 427                 # to find if we can add the closing quotes to the line without
 428                 # exceeding the maximum line length.
 429                 # If docstring is one line, we don't put the closing quotes on a
 430                 # separate line because it looks ugly (#3320).
 431                 lines = docstring.splitlines()
 432                 last_line_length = len(lines[-1]) if docstring else 0
 433
 434                 # If adding closing quotes would cause the last line to exceed
 435                 # the maximum line length then put a line break before the
 436                 # closing quotes
 437                 if (
 438                     len(lines) > 1
 439                     and last_line_length + quote_len > self.mode.line_length
 440                     and len(indent) + quote_len <= self.mode.line_length
 441                     and not has_trailing_backslash
 442                 ):
 443                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 444                 else:
 445                     leaf.value = prefix + quote + docstring + quote
 446             else:
 447                 leaf.value = prefix + quote + docstring + quote
 448
 449         yield from self.visit_default(leaf)
 450
 451     def __post_init__(self) -> None:
 452         """You are in a twisty little maze of passages."""
 453         self.current_line = Line(mode=self.mode)
 454
 455         v = self.visit_stmt
 456         Ø: Set[str] = set()
 457         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 458         self.visit_if_stmt = partial(
 459             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 460         )
 461         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 462         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 463         self.visit_try_stmt = partial(
 464             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 465         )
 466         if self.mode.preview:
 467             self.visit_except_clause = partial(
 468                 v, keywords={"except"}, parens={"except"}
 469             )
 470             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 471         else:
 472             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
 473             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
 474         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 475         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 476         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 477         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 478         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 479         self.visit_async_funcdef = self.visit_async_stmt
 480         self.visit_decorated = self.visit_decorators
 481
 482         # PEP 634
 483         self.visit_match_stmt = self.visit_match_case
 484         self.visit_case_block = self.visit_match_case
 485
 486
 487 def transform_line(
 488     line: Line, mode: Mode, features: Collection[Feature] = ()
 489 ) -> Iterator[Line]:
 490     """Transform a `line`, potentially splitting it into many lines.
 491
 492     They should fit in the allotted `line_length` but might not be able to.
 493
 494     `features` are syntactical features that may be used in the output.
 495     """
 496     if line.is_comment:
 497         yield line
 498         return
 499
 500     line_str = line_to_string(line)
 501
 502     ll = mode.line_length
 503     sn = mode.string_normalization
 504     string_merge = StringMerger(ll, sn)
 505     string_paren_strip = StringParenStripper(ll, sn)
 506     string_split = StringSplitter(ll, sn)
 507     string_paren_wrap = StringParenWrapper(ll, sn)
 508
 509     transformers: List[Transformer]
 510     if (
 511         not line.contains_uncollapsable_type_comments()
 512         and not line.should_split_rhs
 513         and not line.magic_trailing_comma
 514         and (
 515             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
 516             or line.contains_unsplittable_type_ignore()
 517         )
 518         and not (line.inside_brackets and line.contains_standalone_comments())
 519     ):
 520         # Only apply basic string preprocessing, since lines shouldn't be split here.
 521         if Preview.string_processing in mode:
 522             transformers = [string_merge, string_paren_strip]
 523         else:
 524             transformers = []
 525     elif line.is_def:
 526         transformers = [left_hand_split]
 527     else:
 528
 529         def _rhs(
 530             self: object, line: Line, features: Collection[Feature]
 531         ) -> Iterator[Line]:
 532             """Wraps calls to `right_hand_split`.
 533
 534             The calls increasingly `omit` right-hand trailers (bracket pairs with
 535             content), meaning the trailers get glued together to split on another
 536             bracket pair instead.
 537             """
 538             for omit in generate_trailers_to_omit(line, mode.line_length):
 539                 lines = list(
 540                     right_hand_split(line, mode.line_length, features, omit=omit)
 541                 )
 542                 # Note: this check is only able to figure out if the first line of the
 543                 # *current* transformation fits in the line length.  This is true only
 544                 # for simple cases.  All others require running more transforms via
 545                 # `transform_line()`.  This check doesn't know if those would succeed.
 546                 if is_line_short_enough(lines[0], line_length=mode.line_length):
 547                     yield from lines
 548                     return
 549
 550             # All splits failed, best effort split with no omits.
 551             # This mostly happens to multiline strings that are by definition
 552             # reported as not fitting a single line, as well as lines that contain
 553             # trailing commas (those have to be exploded).
 554             yield from right_hand_split(
 555                 line, line_length=mode.line_length, features=features
 556             )
 557
 558         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 559         # __name__ attribute which is needed in `run_transformer` further down.
 560         # Unfortunately a nested class breaks mypyc too. So a class must be created
 561         # via type ... https://github.com/mypyc/mypyc/issues/884
 562         rhs = type("rhs", (), {"__call__": _rhs})()
 563
 564         if Preview.string_processing in mode:
 565             if line.inside_brackets:
 566                 transformers = [
 567                     string_merge,
 568                     string_paren_strip,
 569                     string_split,
 570                     delimiter_split,
 571                     standalone_comment_split,
 572                     string_paren_wrap,
 573                     rhs,
 574                 ]
 575             else:
 576                 transformers = [
 577                     string_merge,
 578                     string_paren_strip,
 579                     string_split,
 580                     string_paren_wrap,
 581                     rhs,
 582                 ]
 583         else:
 584             if line.inside_brackets:
 585                 transformers = [delimiter_split, standalone_comment_split, rhs]
 586             else:
 587                 transformers = [rhs]
 588     # It's always safe to attempt hugging of power operations and pretty much every line
 589     # could match.
 590     transformers.append(hug_power_op)
 591
 592     for transform in transformers:
 593         # We are accumulating lines in `result` because we might want to abort
 594         # mission and return the original line in the end, or attempt a different
 595         # split altogether.
 596         try:
 597             result = run_transformer(line, transform, mode, features, line_str=line_str)
 598         except CannotTransform:
 599             continue
 600         else:
 601             yield from result
 602             break
 603
 604     else:
 605         yield line
 606
 607
 608 class _BracketSplitComponent(Enum):
 609     head = auto()
 610     body = auto()
 611     tail = auto()
 612
 613
 614 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
 615     """Split line into many lines, starting with the first matching bracket pair.
 616
 617     Note: this usually looks weird, only use this for function definitions.
 618     Prefer RHS otherwise.  This is why this function is not symmetrical with
 619     :func:`right_hand_split` which also handles optional parentheses.
 620     """
 621     tail_leaves: List[Leaf] = []
 622     body_leaves: List[Leaf] = []
 623     head_leaves: List[Leaf] = []
 624     current_leaves = head_leaves
 625     matching_bracket: Optional[Leaf] = None
 626     for leaf in line.leaves:
 627         if (
 628             current_leaves is body_leaves
 629             and leaf.type in CLOSING_BRACKETS
 630             and leaf.opening_bracket is matching_bracket
 631             and isinstance(matching_bracket, Leaf)
 632         ):
 633             ensure_visible(leaf)
 634             ensure_visible(matching_bracket)
 635             current_leaves = tail_leaves if body_leaves else head_leaves
 636         current_leaves.append(leaf)
 637         if current_leaves is head_leaves:
 638             if leaf.type in OPENING_BRACKETS:
 639                 matching_bracket = leaf
 640                 current_leaves = body_leaves
 641     if not matching_bracket:
 642         raise CannotSplit("No brackets found")
 643
 644     head = bracket_split_build_line(
 645         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 646     )
 647     body = bracket_split_build_line(
 648         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 649     )
 650     tail = bracket_split_build_line(
 651         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 652     )
 653     bracket_split_succeeded_or_raise(head, body, tail)
 654     for result in (head, body, tail):
 655         if result:
 656             yield result
 657
 658
 659 @dataclass
 660 class _RHSResult:
 661     """Intermediate split result from a right hand split."""
 662
 663     head: Line
 664     body: Line
 665     tail: Line
 666     opening_bracket: Leaf
 667     closing_bracket: Leaf
 668
 669
 670 def right_hand_split(
 671     line: Line,
 672     line_length: int,
 673     features: Collection[Feature] = (),
 674     omit: Collection[LeafID] = (),
 675 ) -> Iterator[Line]:
 676     """Split line into many lines, starting with the last matching bracket pair.
 677
 678     If the split was by optional parentheses, attempt splitting without them, too.
 679     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 680     this split.
 681
 682     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 683     """
 684     rhs_result = _first_right_hand_split(line, omit=omit)
 685     yield from _maybe_split_omitting_optional_parens(
 686         rhs_result, line, line_length, features=features, omit=omit
 687     )
 688
 689
 690 def _first_right_hand_split(
 691     line: Line,
 692     omit: Collection[LeafID] = (),
 693 ) -> _RHSResult:
 694     """Split the line into head, body, tail starting with the last bracket pair.
 695
 696     Note: this function should not have side effects. It's relied upon by
 697     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 698     splitting on the right side of an assignment statement.
 699     """
 700     tail_leaves: List[Leaf] = []
 701     body_leaves: List[Leaf] = []
 702     head_leaves: List[Leaf] = []
 703     current_leaves = tail_leaves
 704     opening_bracket: Optional[Leaf] = None
 705     closing_bracket: Optional[Leaf] = None
 706     for leaf in reversed(line.leaves):
 707         if current_leaves is body_leaves:
 708             if leaf is opening_bracket:
 709                 current_leaves = head_leaves if body_leaves else tail_leaves
 710         current_leaves.append(leaf)
 711         if current_leaves is tail_leaves:
 712             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 713                 opening_bracket = leaf.opening_bracket
 714                 closing_bracket = leaf
 715                 current_leaves = body_leaves
 716     if not (opening_bracket and closing_bracket and head_leaves):
 717         # If there is no opening or closing_bracket that means the split failed and
 718         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 719         # the matching `opening_bracket` wasn't available on `line` anymore.
 720         raise CannotSplit("No brackets found")
 721
 722     tail_leaves.reverse()
 723     body_leaves.reverse()
 724     head_leaves.reverse()
 725     head = bracket_split_build_line(
 726         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 727     )
 728     body = bracket_split_build_line(
 729         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 730     )
 731     tail = bracket_split_build_line(
 732         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 733     )
 734     bracket_split_succeeded_or_raise(head, body, tail)
 735     return _RHSResult(head, body, tail, opening_bracket, closing_bracket)
 736
 737
 738 def _maybe_split_omitting_optional_parens(
 739     rhs: _RHSResult,
 740     line: Line,
 741     line_length: int,
 742     features: Collection[Feature] = (),
 743     omit: Collection[LeafID] = (),
 744 ) -> Iterator[Line]:
 745     if (
 746         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 747         # the opening bracket is an optional paren
 748         and rhs.opening_bracket.type == token.LPAR
 749         and not rhs.opening_bracket.value
 750         # the closing bracket is an optional paren
 751         and rhs.closing_bracket.type == token.RPAR
 752         and not rhs.closing_bracket.value
 753         # it's not an import (optional parens are the only thing we can split on
 754         # in this case; attempting a split without them is a waste of time)
 755         and not line.is_import
 756         # there are no standalone comments in the body
 757         and not rhs.body.contains_standalone_comments(0)
 758         # and we can actually remove the parens
 759         and can_omit_invisible_parens(rhs.body, line_length)
 760     ):
 761         omit = {id(rhs.closing_bracket), *omit}
 762         try:
 763             # The _RHSResult Omitting Optional Parens.
 764             rhs_oop = _first_right_hand_split(line, omit=omit)
 765             if not (
 766                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 767                 # the split is right after `=`
 768                 and len(rhs.head.leaves) >= 2
 769                 and rhs.head.leaves[-2].type == token.EQUAL
 770                 # the left side of assignement contains brackets
 771                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 772                 # the left side of assignment is short enough (the -1 is for the ending
 773                 # optional paren)
 774                 and is_line_short_enough(rhs.head, line_length=line_length - 1)
 775                 # the left side of assignment won't explode further because of magic
 776                 # trailing comma
 777                 and rhs.head.magic_trailing_comma is None
 778                 # the split by omitting optional parens isn't preferred by some other
 779                 # reason
 780                 and not _prefer_split_rhs_oop(rhs_oop, line_length=line_length)
 781             ):
 782                 yield from _maybe_split_omitting_optional_parens(
 783                     rhs_oop, line, line_length, features=features, omit=omit
 784                 )
 785                 return
 786
 787         except CannotSplit as e:
 788             if not (
 789                 can_be_split(rhs.body)
 790                 or is_line_short_enough(rhs.body, line_length=line_length)
 791             ):
 792                 raise CannotSplit(
 793                     "Splitting failed, body is still too long and can't be split."
 794                 ) from e
 795
 796             elif (
 797                 rhs.head.contains_multiline_strings()
 798                 or rhs.tail.contains_multiline_strings()
 799             ):
 800                 raise CannotSplit(
 801                     "The current optional pair of parentheses is bound to fail to"
 802                     " satisfy the splitting algorithm because the head or the tail"
 803                     " contains multiline strings which by definition never fit one"
 804                     " line."
 805                 ) from e
 806
 807     ensure_visible(rhs.opening_bracket)
 808     ensure_visible(rhs.closing_bracket)
 809     for result in (rhs.head, rhs.body, rhs.tail):
 810         if result:
 811             yield result
 812
 813
 814 def _prefer_split_rhs_oop(rhs_oop: _RHSResult, line_length: int) -> bool:
 815     """
 816     Returns whether we should prefer the result from a split omitting optional parens.
 817     """
 818     has_closing_bracket_after_assign = False
 819     for leaf in reversed(rhs_oop.head.leaves):
 820         if leaf.type == token.EQUAL:
 821             break
 822         if leaf.type in CLOSING_BRACKETS:
 823             has_closing_bracket_after_assign = True
 824             break
 825     return (
 826         # contains matching brackets after the `=` (done by checking there is a
 827         # closing bracket)
 828         has_closing_bracket_after_assign
 829         or (
 830             # the split is actually from inside the optional parens (done by checking
 831             # the first line still contains the `=`)
 832             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 833             # the first line is short enough
 834             and is_line_short_enough(rhs_oop.head, line_length=line_length)
 835         )
 836         # contains unsplittable type ignore
 837         or rhs_oop.head.contains_unsplittable_type_ignore()
 838         or rhs_oop.body.contains_unsplittable_type_ignore()
 839         or rhs_oop.tail.contains_unsplittable_type_ignore()
 840     )
 841
 842
 843 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 844     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 845
 846     Do nothing otherwise.
 847
 848     A left- or right-hand split is based on a pair of brackets. Content before
 849     (and including) the opening bracket is left on one line, content inside the
 850     brackets is put on a separate line, and finally content starting with and
 851     following the closing bracket is put on a separate line.
 852
 853     Those are called `head`, `body`, and `tail`, respectively. If the split
 854     produced the same line (all content in `head`) or ended up with an empty `body`
 855     and the `tail` is just the closing bracket, then it's considered failed.
 856     """
 857     tail_len = len(str(tail).strip())
 858     if not body:
 859         if tail_len == 0:
 860             raise CannotSplit("Splitting brackets produced the same line")
 861
 862         elif tail_len < 3:
 863             raise CannotSplit(
 864                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 865                 " not worth it"
 866             )
 867
 868
 869 def bracket_split_build_line(
 870     leaves: List[Leaf],
 871     original: Line,
 872     opening_bracket: Leaf,
 873     *,
 874     component: _BracketSplitComponent,
 875 ) -> Line:
 876     """Return a new line with given `leaves` and respective comments from `original`.
 877
 878     If it's the head component, brackets will be tracked so trailing commas are
 879     respected.
 880
 881     If it's the body component, the result line is one-indented inside brackets and as
 882     such has its first leaf's prefix normalized and a trailing comma added when
 883     expected.
 884     """
 885     result = Line(mode=original.mode, depth=original.depth)
 886     if component is _BracketSplitComponent.body:
 887         result.inside_brackets = True
 888         result.depth += 1
 889         if leaves:
 890             # Since body is a new indent level, remove spurious leading whitespace.
 891             normalize_prefix(leaves[0], inside_brackets=True)
 892             # Ensure a trailing comma for imports and standalone function arguments, but
 893             # be careful not to add one after any comments or within type annotations.
 894             no_commas = (
 895                 original.is_def
 896                 and opening_bracket.value == "("
 897                 and not any(leaf.type == token.COMMA for leaf in leaves)
 898                 # In particular, don't add one within a parenthesized return annotation.
 899                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 900                 # be defined directly in the parent node, the parent of the parent ...
 901                 # and so on depending on how complex the return annotation is.
 902                 # This isn't perfect and there's some false negatives but they are in
 903                 # contexts were a comma is actually fine.
 904                 and not any(
 905                     node.prev_sibling.type == RARROW
 906                     for node in (
 907                         leaves[0].parent,
 908                         getattr(leaves[0].parent, "parent", None),
 909                     )
 910                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 911                 )
 912             )
 913
 914             if original.is_import or no_commas:
 915                 for i in range(len(leaves) - 1, -1, -1):
 916                     if leaves[i].type == STANDALONE_COMMENT:
 917                         continue
 918
 919                     if leaves[i].type != token.COMMA:
 920                         new_comma = Leaf(token.COMMA, ",")
 921                         leaves.insert(i + 1, new_comma)
 922                     break
 923
 924     leaves_to_track: Set[LeafID] = set()
 925     if (
 926         Preview.handle_trailing_commas_in_head in original.mode
 927         and component is _BracketSplitComponent.head
 928     ):
 929         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
 930     # Populate the line
 931     for leaf in leaves:
 932         result.append(
 933             leaf,
 934             preformatted=True,
 935             track_bracket=id(leaf) in leaves_to_track,
 936         )
 937         for comment_after in original.comments_after(leaf):
 938             result.append(comment_after, preformatted=True)
 939     if component is _BracketSplitComponent.body and should_split_line(
 940         result, opening_bracket
 941     ):
 942         result.should_split_rhs = True
 943     return result
 944
 945
 946 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 947     """Normalize prefix of the first leaf in every line returned by `split_func`.
 948
 949     This is a decorator over relevant split functions.
 950     """
 951
 952     @wraps(split_func)
 953     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 954         for split_line in split_func(line, features):
 955             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 956             yield split_line
 957
 958     return split_wrapper
 959
 960
 961 @dont_increase_indentation
 962 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 963     """Split according to delimiters of the highest priority.
 964
 965     If the appropriate Features are given, the split will add trailing commas
 966     also in function signatures and calls that contain `*` and `**`.
 967     """
 968     try:
 969         last_leaf = line.leaves[-1]
 970     except IndexError:
 971         raise CannotSplit("Line empty") from None
 972
 973     bt = line.bracket_tracker
 974     try:
 975         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 976     except ValueError:
 977         raise CannotSplit("No delimiters found") from None
 978
 979     if delimiter_priority == DOT_PRIORITY:
 980         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 981             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 982
 983     current_line = Line(
 984         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 985     )
 986     lowest_depth = sys.maxsize
 987     trailing_comma_safe = True
 988
 989     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 990         """Append `leaf` to current line or to new line if appending impossible."""
 991         nonlocal current_line
 992         try:
 993             current_line.append_safe(leaf, preformatted=True)
 994         except ValueError:
 995             yield current_line
 996
 997             current_line = Line(
 998                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 999             )
1000             current_line.append(leaf)
1001
1002     for leaf in line.leaves:
1003         yield from append_to_line(leaf)
1004
1005         for comment_after in line.comments_after(leaf):
1006             yield from append_to_line(comment_after)
1007
1008         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1009         if leaf.bracket_depth == lowest_depth:
1010             if is_vararg(leaf, within={syms.typedargslist}):
1011                 trailing_comma_safe = (
1012                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1013                 )
1014             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1015                 trailing_comma_safe = (
1016                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1017                 )
1018
1019         leaf_priority = bt.delimiters.get(id(leaf))
1020         if leaf_priority == delimiter_priority:
1021             yield current_line
1022
1023             current_line = Line(
1024                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1025             )
1026     if current_line:
1027         if (
1028             trailing_comma_safe
1029             and delimiter_priority == COMMA_PRIORITY
1030             and current_line.leaves[-1].type != token.COMMA
1031             and current_line.leaves[-1].type != STANDALONE_COMMENT
1032         ):
1033             new_comma = Leaf(token.COMMA, ",")
1034             current_line.append(new_comma)
1035         yield current_line
1036
1037
1038 @dont_increase_indentation
1039 def standalone_comment_split(
1040     line: Line, features: Collection[Feature] = ()
1041 ) -> Iterator[Line]:
1042     """Split standalone comments from the rest of the line."""
1043     if not line.contains_standalone_comments(0):
1044         raise CannotSplit("Line does not have any standalone comments")
1045
1046     current_line = Line(
1047         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1048     )
1049
1050     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1051         """Append `leaf` to current line or to new line if appending impossible."""
1052         nonlocal current_line
1053         try:
1054             current_line.append_safe(leaf, preformatted=True)
1055         except ValueError:
1056             yield current_line
1057
1058             current_line = Line(
1059                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1060             )
1061             current_line.append(leaf)
1062
1063     for leaf in line.leaves:
1064         yield from append_to_line(leaf)
1065
1066         for comment_after in line.comments_after(leaf):
1067             yield from append_to_line(comment_after)
1068
1069     if current_line:
1070         yield current_line
1071
1072
1073 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1074     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1075     else.
1076
1077     Note: don't use backslashes for formatting or you'll lose your voting rights.
1078     """
1079     if not inside_brackets:
1080         spl = leaf.prefix.split("#")
1081         if "\\" not in spl[0]:
1082             nl_count = spl[-1].count("\n")
1083             if len(spl) > 1:
1084                 nl_count -= 1
1085             leaf.prefix = "\n" * nl_count
1086             return
1087
1088     leaf.prefix = ""
1089
1090
1091 def normalize_invisible_parens(
1092     node: Node, parens_after: Set[str], *, preview: bool
1093 ) -> None:
1094     """Make existing optional parentheses invisible or create new ones.
1095
1096     `parens_after` is a set of string leaf values immediately after which parens
1097     should be put.
1098
1099     Standardizes on visible parentheses for single-element tuples, and keeps
1100     existing visible parentheses for other tuples and generator expressions.
1101     """
1102     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
1103         if pc.value in FMT_OFF:
1104             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1105             return
1106     check_lpar = False
1107     for index, child in enumerate(list(node.children)):
1108         # Fixes a bug where invisible parens are not properly stripped from
1109         # assignment statements that contain type annotations.
1110         if isinstance(child, Node) and child.type == syms.annassign:
1111             normalize_invisible_parens(
1112                 child, parens_after=parens_after, preview=preview
1113             )
1114
1115         # Add parentheses around long tuple unpacking in assignments.
1116         if (
1117             index == 0
1118             and isinstance(child, Node)
1119             and child.type == syms.testlist_star_expr
1120         ):
1121             check_lpar = True
1122
1123         if check_lpar:
1124             if (
1125                 preview
1126                 and child.type == syms.atom
1127                 and node.type == syms.for_stmt
1128                 and isinstance(child.prev_sibling, Leaf)
1129                 and child.prev_sibling.type == token.NAME
1130                 and child.prev_sibling.value == "for"
1131             ):
1132                 if maybe_make_parens_invisible_in_atom(
1133                     child,
1134                     parent=node,
1135                     remove_brackets_around_comma=True,
1136                 ):
1137                     wrap_in_parentheses(node, child, visible=False)
1138             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
1139                 remove_with_parens(child, node)
1140             elif child.type == syms.atom:
1141                 if maybe_make_parens_invisible_in_atom(
1142                     child,
1143                     parent=node,
1144                 ):
1145                     wrap_in_parentheses(node, child, visible=False)
1146             elif is_one_tuple(child):
1147                 wrap_in_parentheses(node, child, visible=True)
1148             elif node.type == syms.import_from:
1149                 # "import from" nodes store parentheses directly as part of
1150                 # the statement
1151                 if is_lpar_token(child):
1152                     assert is_rpar_token(node.children[-1])
1153                     # make parentheses invisible
1154                     child.value = ""
1155                     node.children[-1].value = ""
1156                 elif child.type != token.STAR:
1157                     # insert invisible parentheses
1158                     node.insert_child(index, Leaf(token.LPAR, ""))
1159                     node.append_child(Leaf(token.RPAR, ""))
1160                 break
1161             elif (
1162                 index == 1
1163                 and child.type == token.STAR
1164                 and node.type == syms.except_clause
1165             ):
1166                 # In except* (PEP 654), the star is actually part of
1167                 # of the keyword. So we need to skip the insertion of
1168                 # invisible parentheses to work more precisely.
1169                 continue
1170
1171             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1172                 wrap_in_parentheses(node, child, visible=False)
1173
1174         comma_check = child.type == token.COMMA if preview else False
1175
1176         check_lpar = isinstance(child, Leaf) and (
1177             child.value in parens_after or comma_check
1178         )
1179
1180
1181 def remove_await_parens(node: Node) -> None:
1182     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1183         if (
1184             node.children[1].type == syms.atom
1185             and node.children[1].children[0].type == token.LPAR
1186         ):
1187             if maybe_make_parens_invisible_in_atom(
1188                 node.children[1],
1189                 parent=node,
1190                 remove_brackets_around_comma=True,
1191             ):
1192                 wrap_in_parentheses(node, node.children[1], visible=False)
1193
1194             # Since await is an expression we shouldn't remove
1195             # brackets in cases where this would change
1196             # the AST due to operator precedence.
1197             # Therefore we only aim to remove brackets around
1198             # power nodes that aren't also await expressions themselves.
1199             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1200             # N.B. We've still removed any redundant nested brackets though :)
1201             opening_bracket = cast(Leaf, node.children[1].children[0])
1202             closing_bracket = cast(Leaf, node.children[1].children[-1])
1203             bracket_contents = cast(Node, node.children[1].children[1])
1204             if bracket_contents.type != syms.power:
1205                 ensure_visible(opening_bracket)
1206                 ensure_visible(closing_bracket)
1207             elif (
1208                 bracket_contents.type == syms.power
1209                 and bracket_contents.children[0].type == token.AWAIT
1210             ):
1211                 ensure_visible(opening_bracket)
1212                 ensure_visible(closing_bracket)
1213                 # If we are in a nested await then recurse down.
1214                 remove_await_parens(bracket_contents)
1215
1216
1217 def remove_with_parens(node: Node, parent: Node) -> None:
1218     """Recursively hide optional parens in `with` statements."""
1219     # Removing all unnecessary parentheses in with statements in one pass is a tad
1220     # complex as different variations of bracketed statements result in pretty
1221     # different parse trees:
1222     #
1223     # with (open("file")) as f:                       # this is an asexpr_test
1224     #     ...
1225     #
1226     # with (open("file") as f):                       # this is an atom containing an
1227     #     ...                                         # asexpr_test
1228     #
1229     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1230     #     ...                                         # asexpr_test
1231     #
1232     # with (open("file") as f, open("file") as f):    # an atom containing a
1233     #     ...                                         # testlist_gexp which then
1234     #                                                 # contains multiple asexpr_test(s)
1235     if node.type == syms.atom:
1236         if maybe_make_parens_invisible_in_atom(
1237             node,
1238             parent=parent,
1239             remove_brackets_around_comma=True,
1240         ):
1241             wrap_in_parentheses(parent, node, visible=False)
1242         if isinstance(node.children[1], Node):
1243             remove_with_parens(node.children[1], node)
1244     elif node.type == syms.testlist_gexp:
1245         for child in node.children:
1246             if isinstance(child, Node):
1247                 remove_with_parens(child, node)
1248     elif node.type == syms.asexpr_test and not any(
1249         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1250     ):
1251         if maybe_make_parens_invisible_in_atom(
1252             node.children[0],
1253             parent=node,
1254             remove_brackets_around_comma=True,
1255         ):
1256             wrap_in_parentheses(node, node.children[0], visible=False)
1257
1258
1259 def maybe_make_parens_invisible_in_atom(
1260     node: LN,
1261     parent: LN,
1262     remove_brackets_around_comma: bool = False,
1263 ) -> bool:
1264     """If it's safe, make the parens in the atom `node` invisible, recursively.
1265     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1266     as they are redundant.
1267
1268     Returns whether the node should itself be wrapped in invisible parentheses.
1269     """
1270     if (
1271         node.type != syms.atom
1272         or is_empty_tuple(node)
1273         or is_one_tuple(node)
1274         or (is_yield(node) and parent.type != syms.expr_stmt)
1275         or (
1276             # This condition tries to prevent removing non-optional brackets
1277             # around a tuple, however, can be a bit overzealous so we provide
1278             # and option to skip this check for `for` and `with` statements.
1279             not remove_brackets_around_comma
1280             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1281         )
1282     ):
1283         return False
1284
1285     if is_walrus_assignment(node):
1286         if parent.type in [
1287             syms.annassign,
1288             syms.expr_stmt,
1289             syms.assert_stmt,
1290             syms.return_stmt,
1291             syms.except_clause,
1292             syms.funcdef,
1293             # these ones aren't useful to end users, but they do please fuzzers
1294             syms.for_stmt,
1295             syms.del_stmt,
1296         ]:
1297             return False
1298
1299     first = node.children[0]
1300     last = node.children[-1]
1301     if is_lpar_token(first) and is_rpar_token(last):
1302         middle = node.children[1]
1303         # make parentheses invisible
1304         first.value = ""
1305         last.value = ""
1306         maybe_make_parens_invisible_in_atom(
1307             middle,
1308             parent=parent,
1309             remove_brackets_around_comma=remove_brackets_around_comma,
1310         )
1311
1312         if is_atom_with_invisible_parens(middle):
1313             # Strip the invisible parens from `middle` by replacing
1314             # it with the child in-between the invisible parens
1315             middle.replace(middle.children[1])
1316
1317         return False
1318
1319     return True
1320
1321
1322 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1323     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1324
1325     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1326         return False
1327
1328     # We're essentially checking if the body is delimited by commas and there's more
1329     # than one of them (we're excluding the trailing comma and if the delimiter priority
1330     # is still commas, that means there's more).
1331     exclude = set()
1332     trailing_comma = False
1333     try:
1334         last_leaf = line.leaves[-1]
1335         if last_leaf.type == token.COMMA:
1336             trailing_comma = True
1337             exclude.add(id(last_leaf))
1338         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1339     except (IndexError, ValueError):
1340         return False
1341
1342     return max_priority == COMMA_PRIORITY and (
1343         (line.mode.magic_trailing_comma and trailing_comma)
1344         # always explode imports
1345         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1346     )
1347
1348
1349 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1350     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1351
1352     Brackets can be omitted if the entire trailer up to and including
1353     a preceding closing bracket fits in one line.
1354
1355     Yielded sets are cumulative (contain results of previous yields, too).  First
1356     set is empty, unless the line should explode, in which case bracket pairs until
1357     the one that needs to explode are omitted.
1358     """
1359
1360     omit: Set[LeafID] = set()
1361     if not line.magic_trailing_comma:
1362         yield omit
1363
1364     length = 4 * line.depth
1365     opening_bracket: Optional[Leaf] = None
1366     closing_bracket: Optional[Leaf] = None
1367     inner_brackets: Set[LeafID] = set()
1368     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1369         length += leaf_length
1370         if length > line_length:
1371             break
1372
1373         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1374         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1375             break
1376
1377         if opening_bracket:
1378             if leaf is opening_bracket:
1379                 opening_bracket = None
1380             elif leaf.type in CLOSING_BRACKETS:
1381                 prev = line.leaves[index - 1] if index > 0 else None
1382                 if (
1383                     prev
1384                     and prev.type == token.COMMA
1385                     and leaf.opening_bracket is not None
1386                     and not is_one_sequence_between(
1387                         leaf.opening_bracket, leaf, line.leaves
1388                     )
1389                 ):
1390                     # Never omit bracket pairs with trailing commas.
1391                     # We need to explode on those.
1392                     break
1393
1394                 inner_brackets.add(id(leaf))
1395         elif leaf.type in CLOSING_BRACKETS:
1396             prev = line.leaves[index - 1] if index > 0 else None
1397             if prev and prev.type in OPENING_BRACKETS:
1398                 # Empty brackets would fail a split so treat them as "inner"
1399                 # brackets (e.g. only add them to the `omit` set if another
1400                 # pair of brackets was good enough.
1401                 inner_brackets.add(id(leaf))
1402                 continue
1403
1404             if closing_bracket:
1405                 omit.add(id(closing_bracket))
1406                 omit.update(inner_brackets)
1407                 inner_brackets.clear()
1408                 yield omit
1409
1410             if (
1411                 prev
1412                 and prev.type == token.COMMA
1413                 and leaf.opening_bracket is not None
1414                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1415             ):
1416                 # Never omit bracket pairs with trailing commas.
1417                 # We need to explode on those.
1418                 break
1419
1420             if leaf.value:
1421                 opening_bracket = leaf.opening_bracket
1422                 closing_bracket = leaf
1423
1424
1425 def run_transformer(
1426     line: Line,
1427     transform: Transformer,
1428     mode: Mode,
1429     features: Collection[Feature],
1430     *,
1431     line_str: str = "",
1432 ) -> List[Line]:
1433     if not line_str:
1434         line_str = line_to_string(line)
1435     result: List[Line] = []
1436     for transformed_line in transform(line, features):
1437         if str(transformed_line).strip("\n") == line_str:
1438             raise CannotTransform("Line transformer returned an unchanged result")
1439
1440         result.extend(transform_line(transformed_line, mode=mode, features=features))
1441
1442     features_set = set(features)
1443     if (
1444         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1445         or transform.__class__.__name__ != "rhs"
1446         or not line.bracket_tracker.invisible
1447         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1448         or line.contains_multiline_strings()
1449         or result[0].contains_uncollapsable_type_comments()
1450         or result[0].contains_unsplittable_type_ignore()
1451         or is_line_short_enough(result[0], line_length=mode.line_length)
1452         # If any leaves have no parents (which _can_ occur since
1453         # `transform(line)` potentially destroys the line's underlying node
1454         # structure), then we can't proceed. Doing so would cause the below
1455         # call to `append_leaves()` to fail.
1456         or any(leaf.parent is None for leaf in line.leaves)
1457     ):
1458         return result
1459
1460     line_copy = line.clone()
1461     append_leaves(line_copy, line, line.leaves)
1462     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1463     second_opinion = run_transformer(
1464         line_copy, transform, mode, features_fop, line_str=line_str
1465     )
1466     if all(
1467         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1468     ):
1469         result = second_opinion
1470     return result