src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import replace
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     RHSResult,
  20     append_leaves,
  21     can_be_split,
  22     can_omit_invisible_parens,
  23     is_line_short_enough,
  24     line_to_string,
  25 )
  26 from black.mode import Feature, Mode, Preview
  27 from black.nodes import (
  28     ASSIGNMENTS,
  29     BRACKETS,
  30     CLOSING_BRACKETS,
  31     OPENING_BRACKETS,
  32     RARROW,
  33     STANDALONE_COMMENT,
  34     STATEMENT,
  35     WHITESPACE,
  36     Visitor,
  37     ensure_visible,
  38     is_arith_like,
  39     is_async_stmt_or_funcdef,
  40     is_atom_with_invisible_parens,
  41     is_docstring,
  42     is_empty_tuple,
  43     is_lpar_token,
  44     is_multiline_string,
  45     is_name_token,
  46     is_one_sequence_between,
  47     is_one_tuple,
  48     is_rpar_token,
  49     is_stub_body,
  50     is_stub_suite,
  51     is_tuple_containing_walrus,
  52     is_type_ignore_comment_string,
  53     is_vararg,
  54     is_walrus_assignment,
  55     is_yield,
  56     syms,
  57     wrap_in_parentheses,
  58 )
  59 from black.numerics import normalize_numeric_literal
  60 from black.strings import (
  61     fix_docstring,
  62     get_string_prefix,
  63     normalize_string_prefix,
  64     normalize_string_quotes,
  65     normalize_unicode_escape_sequences,
  66 )
  67 from black.trans import (
  68     CannotTransform,
  69     StringMerger,
  70     StringParenStripper,
  71     StringParenWrapper,
  72     StringSplitter,
  73     Transformer,
  74     hug_power_op,
  75 )
  76 from blib2to3.pgen2 import token
  77 from blib2to3.pytree import Leaf, Node
  78
  79 # types
  80 LeafID = int
  81 LN = Union[Leaf, Node]
  82
  83
  84 class CannotSplit(CannotTransform):
  85     """A readable split that fits the allotted line length is impossible."""
  86
  87
  88 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  89 # See also https://github.com/mypyc/mypyc/issues/827.
  90 class LineGenerator(Visitor[Line]):
  91     """Generates reformatted Line objects.  Empty lines are not emitted.
  92
  93     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  94     in ways that will no longer stringify to valid Python code on the tree.
  95     """
  96
  97     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  98         self.mode = mode
  99         self.features = features
 100         self.current_line: Line
 101         self.__post_init__()
 102
 103     def line(self, indent: int = 0) -> Iterator[Line]:
 104         """Generate a line.
 105
 106         If the line is empty, only emit if it makes sense.
 107         If the line is too long, split it first and then generate.
 108
 109         If any lines were generated, set up a new current_line.
 110         """
 111         if not self.current_line:
 112             self.current_line.depth += indent
 113             return  # Line is empty, don't emit. Creating a new one unnecessary.
 114
 115         if (
 116             Preview.improved_async_statements_handling in self.mode
 117             and len(self.current_line.leaves) == 1
 118             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
 119         ):
 120             # Special case for async def/for/with statements. `visit_async_stmt`
 121             # adds an `ASYNC` leaf then visits the child def/for/with statement
 122             # nodes. Line yields from those nodes shouldn't treat the former
 123             # `ASYNC` leaf as a complete line.
 124             return
 125
 126         complete_line = self.current_line
 127         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 128         yield complete_line
 129
 130     def visit_default(self, node: LN) -> Iterator[Line]:
 131         """Default `visit_*()` implementation. Recurses to children of `node`."""
 132         if isinstance(node, Leaf):
 133             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 134             for comment in generate_comments(node):
 135                 if any_open_brackets:
 136                     # any comment within brackets is subject to splitting
 137                     self.current_line.append(comment)
 138                 elif comment.type == token.COMMENT:
 139                     # regular trailing comment
 140                     self.current_line.append(comment)
 141                     yield from self.line()
 142
 143                 else:
 144                     # regular standalone comment
 145                     yield from self.line()
 146
 147                     self.current_line.append(comment)
 148                     yield from self.line()
 149
 150             normalize_prefix(node, inside_brackets=any_open_brackets)
 151             if self.mode.string_normalization and node.type == token.STRING:
 152                 node.value = normalize_string_prefix(node.value)
 153                 node.value = normalize_string_quotes(node.value)
 154             if node.type == token.NUMBER:
 155                 normalize_numeric_literal(node)
 156             if node.type not in WHITESPACE:
 157                 self.current_line.append(node)
 158         yield from super().visit_default(node)
 159
 160     def visit_test(self, node: Node) -> Iterator[Line]:
 161         """Visit an `x if y else z` test"""
 162
 163         if Preview.parenthesize_conditional_expressions in self.mode:
 164             already_parenthesized = (
 165                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 166             )
 167
 168             if not already_parenthesized:
 169                 lpar = Leaf(token.LPAR, "")
 170                 rpar = Leaf(token.RPAR, "")
 171                 node.insert_child(0, lpar)
 172                 node.append_child(rpar)
 173
 174         yield from self.visit_default(node)
 175
 176     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 177         """Increase indentation level, maybe yield a line."""
 178         # In blib2to3 INDENT never holds comments.
 179         yield from self.line(+1)
 180         yield from self.visit_default(node)
 181
 182     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 183         """Decrease indentation level, maybe yield a line."""
 184         # The current line might still wait for trailing comments.  At DEDENT time
 185         # there won't be any (they would be prefixes on the preceding NEWLINE).
 186         # Emit the line then.
 187         yield from self.line()
 188
 189         # While DEDENT has no value, its prefix may contain standalone comments
 190         # that belong to the current indentation level.  Get 'em.
 191         yield from self.visit_default(node)
 192
 193         # Finally, emit the dedent.
 194         yield from self.line(-1)
 195
 196     def visit_stmt(
 197         self, node: Node, keywords: Set[str], parens: Set[str]
 198     ) -> Iterator[Line]:
 199         """Visit a statement.
 200
 201         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 202         `def`, `with`, `class`, `assert`, and assignments.
 203
 204         The relevant Python language `keywords` for a given statement will be
 205         NAME leaves within it. This methods puts those on a separate line.
 206
 207         `parens` holds a set of string leaf values immediately after which
 208         invisible parens should be put.
 209         """
 210         normalize_invisible_parens(
 211             node, parens_after=parens, mode=self.mode, features=self.features
 212         )
 213         for child in node.children:
 214             if is_name_token(child) and child.value in keywords:
 215                 yield from self.line()
 216
 217             yield from self.visit(child)
 218
 219     def visit_typeparams(self, node: Node) -> Iterator[Line]:
 220         yield from self.visit_default(node)
 221         node.children[0].prefix = ""
 222
 223     def visit_typevartuple(self, node: Node) -> Iterator[Line]:
 224         yield from self.visit_default(node)
 225         node.children[1].prefix = ""
 226
 227     def visit_paramspec(self, node: Node) -> Iterator[Line]:
 228         yield from self.visit_default(node)
 229         node.children[1].prefix = ""
 230
 231     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 232         if Preview.wrap_long_dict_values_in_parens in self.mode:
 233             for i, child in enumerate(node.children):
 234                 if i == 0:
 235                     continue
 236                 if node.children[i - 1].type == token.COLON:
 237                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 238                         if maybe_make_parens_invisible_in_atom(
 239                             child,
 240                             parent=node,
 241                             remove_brackets_around_comma=False,
 242                         ):
 243                             wrap_in_parentheses(node, child, visible=False)
 244                     else:
 245                         wrap_in_parentheses(node, child, visible=False)
 246         yield from self.visit_default(node)
 247
 248     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 249         """Visit function definition."""
 250         yield from self.line()
 251
 252         # Remove redundant brackets around return type annotation.
 253         is_return_annotation = False
 254         for child in node.children:
 255             if child.type == token.RARROW:
 256                 is_return_annotation = True
 257             elif is_return_annotation:
 258                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 259                     if maybe_make_parens_invisible_in_atom(
 260                         child,
 261                         parent=node,
 262                         remove_brackets_around_comma=False,
 263                     ):
 264                         wrap_in_parentheses(node, child, visible=False)
 265                 else:
 266                     wrap_in_parentheses(node, child, visible=False)
 267                 is_return_annotation = False
 268
 269         for child in node.children:
 270             yield from self.visit(child)
 271
 272     def visit_match_case(self, node: Node) -> Iterator[Line]:
 273         """Visit either a match or case statement."""
 274         normalize_invisible_parens(
 275             node, parens_after=set(), mode=self.mode, features=self.features
 276         )
 277
 278         yield from self.line()
 279         for child in node.children:
 280             yield from self.visit(child)
 281
 282     def visit_suite(self, node: Node) -> Iterator[Line]:
 283         """Visit a suite."""
 284         if (
 285             self.mode.is_pyi or Preview.dummy_implementations in self.mode
 286         ) and is_stub_suite(node):
 287             yield from self.visit(node.children[2])
 288         else:
 289             yield from self.visit_default(node)
 290
 291     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 292         """Visit a statement without nested statements."""
 293         prev_type: Optional[int] = None
 294         for child in node.children:
 295             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 296                 wrap_in_parentheses(node, child, visible=False)
 297             prev_type = child.type
 298
 299         is_suite_like = node.parent and node.parent.type in STATEMENT
 300         if is_suite_like:
 301             if (
 302                 self.mode.is_pyi or Preview.dummy_implementations in self.mode
 303             ) and is_stub_body(node):
 304                 yield from self.visit_default(node)
 305             else:
 306                 yield from self.line(+1)
 307                 yield from self.visit_default(node)
 308                 yield from self.line(-1)
 309
 310         else:
 311             if (
 312                 not (self.mode.is_pyi or Preview.dummy_implementations in self.mode)
 313                 or not node.parent
 314                 or not is_stub_suite(node.parent)
 315             ):
 316                 yield from self.line()
 317             yield from self.visit_default(node)
 318
 319     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 320         """Visit `async def`, `async for`, `async with`."""
 321         yield from self.line()
 322
 323         children = iter(node.children)
 324         for child in children:
 325             yield from self.visit(child)
 326
 327             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 328                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 329                 # line.
 330                 break
 331
 332         internal_stmt = next(children)
 333         if Preview.improved_async_statements_handling in self.mode:
 334             yield from self.visit(internal_stmt)
 335         else:
 336             for child in internal_stmt.children:
 337                 yield from self.visit(child)
 338
 339     def visit_decorators(self, node: Node) -> Iterator[Line]:
 340         """Visit decorators."""
 341         for child in node.children:
 342             yield from self.line()
 343             yield from self.visit(child)
 344
 345     def visit_power(self, node: Node) -> Iterator[Line]:
 346         for idx, leaf in enumerate(node.children[:-1]):
 347             next_leaf = node.children[idx + 1]
 348
 349             if not isinstance(leaf, Leaf):
 350                 continue
 351
 352             value = leaf.value.lower()
 353             if (
 354                 leaf.type == token.NUMBER
 355                 and next_leaf.type == syms.trailer
 356                 # Ensure that we are in an attribute trailer
 357                 and next_leaf.children[0].type == token.DOT
 358                 # It shouldn't wrap hexadecimal, binary and octal literals
 359                 and not value.startswith(("0x", "0b", "0o"))
 360                 # It shouldn't wrap complex literals
 361                 and "j" not in value
 362             ):
 363                 wrap_in_parentheses(node, leaf)
 364
 365         remove_await_parens(node)
 366
 367         yield from self.visit_default(node)
 368
 369     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 370         """Remove a semicolon and put the other statement on a separate line."""
 371         yield from self.line()
 372
 373     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 374         """End of file. Process outstanding comments and end with a newline."""
 375         yield from self.visit_default(leaf)
 376         yield from self.line()
 377
 378     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 379         if not self.current_line.bracket_tracker.any_open_brackets():
 380             yield from self.line()
 381         yield from self.visit_default(leaf)
 382
 383     def visit_factor(self, node: Node) -> Iterator[Line]:
 384         """Force parentheses between a unary op and a binary power:
 385
 386         -2 ** 8 -> -(2 ** 8)
 387         """
 388         _operator, operand = node.children
 389         if (
 390             operand.type == syms.power
 391             and len(operand.children) == 3
 392             and operand.children[1].type == token.DOUBLESTAR
 393         ):
 394             lpar = Leaf(token.LPAR, "(")
 395             rpar = Leaf(token.RPAR, ")")
 396             index = operand.remove() or 0
 397             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 398         yield from self.visit_default(node)
 399
 400     def visit_tname(self, node: Node) -> Iterator[Line]:
 401         """
 402         Add potential parentheses around types in function parameter lists to be made
 403         into real parentheses in case the type hint is too long to fit on a line
 404         Examples:
 405         def foo(a: int, b: float = 7): ...
 406
 407         ->
 408
 409         def foo(a: (int), b: (float) = 7): ...
 410         """
 411         if Preview.parenthesize_long_type_hints in self.mode:
 412             assert len(node.children) == 3
 413             if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):
 414                 wrap_in_parentheses(node, node.children[2], visible=False)
 415
 416         yield from self.visit_default(node)
 417
 418     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 419         if Preview.hex_codes_in_unicode_sequences in self.mode:
 420             normalize_unicode_escape_sequences(leaf)
 421
 422         if is_docstring(leaf) and "\\\n" not in leaf.value:
 423             # We're ignoring docstrings with backslash newline escapes because changing
 424             # indentation of those changes the AST representation of the code.
 425             if self.mode.string_normalization:
 426                 docstring = normalize_string_prefix(leaf.value)
 427                 # visit_default() does handle string normalization for us, but
 428                 # since this method acts differently depending on quote style (ex.
 429                 # see padding logic below), there's a possibility for unstable
 430                 # formatting as visit_default() is called *after*. To avoid a
 431                 # situation where this function formats a docstring differently on
 432                 # the second pass, normalize it early.
 433                 docstring = normalize_string_quotes(docstring)
 434             else:
 435                 docstring = leaf.value
 436             prefix = get_string_prefix(docstring)
 437             docstring = docstring[len(prefix) :]  # Remove the prefix
 438             quote_char = docstring[0]
 439             # A natural way to remove the outer quotes is to do:
 440             #   docstring = docstring.strip(quote_char)
 441             # but that breaks on """""x""" (which is '""x').
 442             # So we actually need to remove the first character and the next two
 443             # characters but only if they are the same as the first.
 444             quote_len = 1 if docstring[1] != quote_char else 3
 445             docstring = docstring[quote_len:-quote_len]
 446             docstring_started_empty = not docstring
 447             indent = " " * 4 * self.current_line.depth
 448
 449             if is_multiline_string(leaf):
 450                 docstring = fix_docstring(docstring, indent)
 451             else:
 452                 docstring = docstring.strip()
 453
 454             has_trailing_backslash = False
 455             if docstring:
 456                 # Add some padding if the docstring starts / ends with a quote mark.
 457                 if docstring[0] == quote_char:
 458                     docstring = " " + docstring
 459                 if docstring[-1] == quote_char:
 460                     docstring += " "
 461                 if docstring[-1] == "\\":
 462                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 463                     if backslash_count % 2:
 464                         # Odd number of tailing backslashes, add some padding to
 465                         # avoid escaping the closing string quote.
 466                         docstring += " "
 467                         has_trailing_backslash = True
 468             elif not docstring_started_empty:
 469                 docstring = " "
 470
 471             # We could enforce triple quotes at this point.
 472             quote = quote_char * quote_len
 473
 474             # It's invalid to put closing single-character quotes on a new line.
 475             if self.mode and quote_len == 3:
 476                 # We need to find the length of the last line of the docstring
 477                 # to find if we can add the closing quotes to the line without
 478                 # exceeding the maximum line length.
 479                 # If docstring is one line, we don't put the closing quotes on a
 480                 # separate line because it looks ugly (#3320).
 481                 lines = docstring.splitlines()
 482                 last_line_length = len(lines[-1]) if docstring else 0
 483
 484                 # If adding closing quotes would cause the last line to exceed
 485                 # the maximum line length then put a line break before the
 486                 # closing quotes
 487                 if (
 488                     len(lines) > 1
 489                     and last_line_length + quote_len > self.mode.line_length
 490                     and len(indent) + quote_len <= self.mode.line_length
 491                     and not has_trailing_backslash
 492                 ):
 493                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 494                 else:
 495                     leaf.value = prefix + quote + docstring + quote
 496             else:
 497                 leaf.value = prefix + quote + docstring + quote
 498
 499         yield from self.visit_default(leaf)
 500
 501     def __post_init__(self) -> None:
 502         """You are in a twisty little maze of passages."""
 503         self.current_line = Line(mode=self.mode)
 504
 505         v = self.visit_stmt
 506         Ø: Set[str] = set()
 507         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 508         self.visit_if_stmt = partial(
 509             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 510         )
 511         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 512         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 513         self.visit_try_stmt = partial(
 514             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 515         )
 516         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 517         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 518         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 519
 520         # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py
 521         if Preview.parenthesize_long_type_hints in self.mode:
 522             assignments = ASSIGNMENTS | {":"}
 523         else:
 524             assignments = ASSIGNMENTS
 525         self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments)
 526
 527         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 528         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 529         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 530         self.visit_async_funcdef = self.visit_async_stmt
 531         self.visit_decorated = self.visit_decorators
 532
 533         # PEP 634
 534         self.visit_match_stmt = self.visit_match_case
 535         self.visit_case_block = self.visit_match_case
 536
 537
 538 def transform_line(
 539     line: Line, mode: Mode, features: Collection[Feature] = ()
 540 ) -> Iterator[Line]:
 541     """Transform a `line`, potentially splitting it into many lines.
 542
 543     They should fit in the allotted `line_length` but might not be able to.
 544
 545     `features` are syntactical features that may be used in the output.
 546     """
 547     if line.is_comment:
 548         yield line
 549         return
 550
 551     line_str = line_to_string(line)
 552
 553     ll = mode.line_length
 554     sn = mode.string_normalization
 555     string_merge = StringMerger(ll, sn)
 556     string_paren_strip = StringParenStripper(ll, sn)
 557     string_split = StringSplitter(ll, sn)
 558     string_paren_wrap = StringParenWrapper(ll, sn)
 559
 560     transformers: List[Transformer]
 561     if (
 562         not line.contains_uncollapsable_type_comments()
 563         and not line.should_split_rhs
 564         and not line.magic_trailing_comma
 565         and (
 566             is_line_short_enough(line, mode=mode, line_str=line_str)
 567             or line.contains_unsplittable_type_ignore()
 568         )
 569         and not (line.inside_brackets and line.contains_standalone_comments())
 570     ):
 571         # Only apply basic string preprocessing, since lines shouldn't be split here.
 572         if Preview.string_processing in mode:
 573             transformers = [string_merge, string_paren_strip]
 574         else:
 575             transformers = []
 576     elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
 577         transformers = [left_hand_split]
 578     else:
 579
 580         def _rhs(
 581             self: object, line: Line, features: Collection[Feature], mode: Mode
 582         ) -> Iterator[Line]:
 583             """Wraps calls to `right_hand_split`.
 584
 585             The calls increasingly `omit` right-hand trailers (bracket pairs with
 586             content), meaning the trailers get glued together to split on another
 587             bracket pair instead.
 588             """
 589             for omit in generate_trailers_to_omit(line, mode.line_length):
 590                 lines = list(right_hand_split(line, mode, features, omit=omit))
 591                 # Note: this check is only able to figure out if the first line of the
 592                 # *current* transformation fits in the line length.  This is true only
 593                 # for simple cases.  All others require running more transforms via
 594                 # `transform_line()`.  This check doesn't know if those would succeed.
 595                 if is_line_short_enough(lines[0], mode=mode):
 596                     yield from lines
 597                     return
 598
 599             # All splits failed, best effort split with no omits.
 600             # This mostly happens to multiline strings that are by definition
 601             # reported as not fitting a single line, as well as lines that contain
 602             # trailing commas (those have to be exploded).
 603             yield from right_hand_split(line, mode, features=features)
 604
 605         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 606         # __name__ attribute which is needed in `run_transformer` further down.
 607         # Unfortunately a nested class breaks mypyc too. So a class must be created
 608         # via type ... https://github.com/mypyc/mypyc/issues/884
 609         rhs = type("rhs", (), {"__call__": _rhs})()
 610
 611         if Preview.string_processing in mode:
 612             if line.inside_brackets:
 613                 transformers = [
 614                     string_merge,
 615                     string_paren_strip,
 616                     string_split,
 617                     delimiter_split,
 618                     standalone_comment_split,
 619                     string_paren_wrap,
 620                     rhs,
 621                 ]
 622             else:
 623                 transformers = [
 624                     string_merge,
 625                     string_paren_strip,
 626                     string_split,
 627                     string_paren_wrap,
 628                     rhs,
 629                 ]
 630         else:
 631             if line.inside_brackets:
 632                 transformers = [delimiter_split, standalone_comment_split, rhs]
 633             else:
 634                 transformers = [rhs]
 635     # It's always safe to attempt hugging of power operations and pretty much every line
 636     # could match.
 637     transformers.append(hug_power_op)
 638
 639     for transform in transformers:
 640         # We are accumulating lines in `result` because we might want to abort
 641         # mission and return the original line in the end, or attempt a different
 642         # split altogether.
 643         try:
 644             result = run_transformer(line, transform, mode, features, line_str=line_str)
 645         except CannotTransform:
 646             continue
 647         else:
 648             yield from result
 649             break
 650
 651     else:
 652         yield line
 653
 654
 655 def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
 656     """If a funcdef has a magic trailing comma in the return type, then we should first
 657     split the line with rhs to respect the comma.
 658     """
 659     if Preview.respect_magic_trailing_comma_in_return_type not in mode:
 660         return False
 661
 662     return_type_leaves: List[Leaf] = []
 663     in_return_type = False
 664
 665     for leaf in line.leaves:
 666         if leaf.type == token.COLON:
 667             in_return_type = False
 668         if in_return_type:
 669             return_type_leaves.append(leaf)
 670         if leaf.type == token.RARROW:
 671             in_return_type = True
 672
 673     # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
 674     # couple lines from it.
 675     result = Line(mode=line.mode, depth=line.depth)
 676     leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
 677     for leaf in return_type_leaves:
 678         result.append(
 679             leaf,
 680             preformatted=True,
 681             track_bracket=id(leaf) in leaves_to_track,
 682         )
 683
 684     # we could also return true if the line is too long, and the return type is longer
 685     # than the param list. Or if `should_split_rhs` returns True.
 686     return result.magic_trailing_comma is not None
 687
 688
 689 class _BracketSplitComponent(Enum):
 690     head = auto()
 691     body = auto()
 692     tail = auto()
 693
 694
 695 def left_hand_split(
 696     line: Line, _features: Collection[Feature], mode: Mode
 697 ) -> Iterator[Line]:
 698     """Split line into many lines, starting with the first matching bracket pair.
 699
 700     Note: this usually looks weird, only use this for function definitions.
 701     Prefer RHS otherwise.  This is why this function is not symmetrical with
 702     :func:`right_hand_split` which also handles optional parentheses.
 703     """
 704     tail_leaves: List[Leaf] = []
 705     body_leaves: List[Leaf] = []
 706     head_leaves: List[Leaf] = []
 707     current_leaves = head_leaves
 708     matching_bracket: Optional[Leaf] = None
 709     for leaf in line.leaves:
 710         if (
 711             current_leaves is body_leaves
 712             and leaf.type in CLOSING_BRACKETS
 713             and leaf.opening_bracket is matching_bracket
 714             and isinstance(matching_bracket, Leaf)
 715         ):
 716             ensure_visible(leaf)
 717             ensure_visible(matching_bracket)
 718             current_leaves = tail_leaves if body_leaves else head_leaves
 719         current_leaves.append(leaf)
 720         if current_leaves is head_leaves:
 721             if leaf.type in OPENING_BRACKETS:
 722                 matching_bracket = leaf
 723                 current_leaves = body_leaves
 724     if not matching_bracket:
 725         raise CannotSplit("No brackets found")
 726
 727     head = bracket_split_build_line(
 728         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 729     )
 730     body = bracket_split_build_line(
 731         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 732     )
 733     tail = bracket_split_build_line(
 734         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 735     )
 736     bracket_split_succeeded_or_raise(head, body, tail)
 737     for result in (head, body, tail):
 738         if result:
 739             yield result
 740
 741
 742 def right_hand_split(
 743     line: Line,
 744     mode: Mode,
 745     features: Collection[Feature] = (),
 746     omit: Collection[LeafID] = (),
 747 ) -> Iterator[Line]:
 748     """Split line into many lines, starting with the last matching bracket pair.
 749
 750     If the split was by optional parentheses, attempt splitting without them, too.
 751     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 752     this split.
 753
 754     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 755     """
 756     rhs_result = _first_right_hand_split(line, omit=omit)
 757     yield from _maybe_split_omitting_optional_parens(
 758         rhs_result, line, mode, features=features, omit=omit
 759     )
 760
 761
 762 def _first_right_hand_split(
 763     line: Line,
 764     omit: Collection[LeafID] = (),
 765 ) -> RHSResult:
 766     """Split the line into head, body, tail starting with the last bracket pair.
 767
 768     Note: this function should not have side effects. It's relied upon by
 769     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 770     splitting on the right side of an assignment statement.
 771     """
 772     tail_leaves: List[Leaf] = []
 773     body_leaves: List[Leaf] = []
 774     head_leaves: List[Leaf] = []
 775     current_leaves = tail_leaves
 776     opening_bracket: Optional[Leaf] = None
 777     closing_bracket: Optional[Leaf] = None
 778     for leaf in reversed(line.leaves):
 779         if current_leaves is body_leaves:
 780             if leaf is opening_bracket:
 781                 current_leaves = head_leaves if body_leaves else tail_leaves
 782         current_leaves.append(leaf)
 783         if current_leaves is tail_leaves:
 784             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 785                 opening_bracket = leaf.opening_bracket
 786                 closing_bracket = leaf
 787                 current_leaves = body_leaves
 788     if not (opening_bracket and closing_bracket and head_leaves):
 789         # If there is no opening or closing_bracket that means the split failed and
 790         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 791         # the matching `opening_bracket` wasn't available on `line` anymore.
 792         raise CannotSplit("No brackets found")
 793
 794     tail_leaves.reverse()
 795     body_leaves.reverse()
 796     head_leaves.reverse()
 797     head = bracket_split_build_line(
 798         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 799     )
 800     body = bracket_split_build_line(
 801         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 802     )
 803     tail = bracket_split_build_line(
 804         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 805     )
 806     bracket_split_succeeded_or_raise(head, body, tail)
 807     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 808
 809
 810 def _maybe_split_omitting_optional_parens(
 811     rhs: RHSResult,
 812     line: Line,
 813     mode: Mode,
 814     features: Collection[Feature] = (),
 815     omit: Collection[LeafID] = (),
 816 ) -> Iterator[Line]:
 817     if (
 818         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 819         # the opening bracket is an optional paren
 820         and rhs.opening_bracket.type == token.LPAR
 821         and not rhs.opening_bracket.value
 822         # the closing bracket is an optional paren
 823         and rhs.closing_bracket.type == token.RPAR
 824         and not rhs.closing_bracket.value
 825         # it's not an import (optional parens are the only thing we can split on
 826         # in this case; attempting a split without them is a waste of time)
 827         and not line.is_import
 828         # there are no standalone comments in the body
 829         and not rhs.body.contains_standalone_comments(0)
 830         # and we can actually remove the parens
 831         and can_omit_invisible_parens(rhs, mode.line_length)
 832     ):
 833         omit = {id(rhs.closing_bracket), *omit}
 834         try:
 835             # The RHSResult Omitting Optional Parens.
 836             rhs_oop = _first_right_hand_split(line, omit=omit)
 837             if not (
 838                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 839                 # the split is right after `=`
 840                 and len(rhs.head.leaves) >= 2
 841                 and rhs.head.leaves[-2].type == token.EQUAL
 842                 # the left side of assignment contains brackets
 843                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 844                 # the left side of assignment is short enough (the -1 is for the ending
 845                 # optional paren)
 846                 and is_line_short_enough(
 847                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 848                 )
 849                 # the left side of assignment won't explode further because of magic
 850                 # trailing comma
 851                 and rhs.head.magic_trailing_comma is None
 852                 # the split by omitting optional parens isn't preferred by some other
 853                 # reason
 854                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 855             ):
 856                 yield from _maybe_split_omitting_optional_parens(
 857                     rhs_oop, line, mode, features=features, omit=omit
 858                 )
 859                 return
 860
 861         except CannotSplit as e:
 862             if not (
 863                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 864             ):
 865                 raise CannotSplit(
 866                     "Splitting failed, body is still too long and can't be split."
 867                 ) from e
 868
 869             elif (
 870                 rhs.head.contains_multiline_strings()
 871                 or rhs.tail.contains_multiline_strings()
 872             ):
 873                 raise CannotSplit(
 874                     "The current optional pair of parentheses is bound to fail to"
 875                     " satisfy the splitting algorithm because the head or the tail"
 876                     " contains multiline strings which by definition never fit one"
 877                     " line."
 878                 ) from e
 879
 880     ensure_visible(rhs.opening_bracket)
 881     ensure_visible(rhs.closing_bracket)
 882     for result in (rhs.head, rhs.body, rhs.tail):
 883         if result:
 884             yield result
 885
 886
 887 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 888     """
 889     Returns whether we should prefer the result from a split omitting optional parens.
 890     """
 891     has_closing_bracket_after_assign = False
 892     for leaf in reversed(rhs_oop.head.leaves):
 893         if leaf.type == token.EQUAL:
 894             break
 895         if leaf.type in CLOSING_BRACKETS:
 896             has_closing_bracket_after_assign = True
 897             break
 898     return (
 899         # contains matching brackets after the `=` (done by checking there is a
 900         # closing bracket)
 901         has_closing_bracket_after_assign
 902         or (
 903             # the split is actually from inside the optional parens (done by checking
 904             # the first line still contains the `=`)
 905             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 906             # the first line is short enough
 907             and is_line_short_enough(rhs_oop.head, mode=mode)
 908         )
 909         # contains unsplittable type ignore
 910         or rhs_oop.head.contains_unsplittable_type_ignore()
 911         or rhs_oop.body.contains_unsplittable_type_ignore()
 912         or rhs_oop.tail.contains_unsplittable_type_ignore()
 913     )
 914
 915
 916 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 917     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 918
 919     Do nothing otherwise.
 920
 921     A left- or right-hand split is based on a pair of brackets. Content before
 922     (and including) the opening bracket is left on one line, content inside the
 923     brackets is put on a separate line, and finally content starting with and
 924     following the closing bracket is put on a separate line.
 925
 926     Those are called `head`, `body`, and `tail`, respectively. If the split
 927     produced the same line (all content in `head`) or ended up with an empty `body`
 928     and the `tail` is just the closing bracket, then it's considered failed.
 929     """
 930     tail_len = len(str(tail).strip())
 931     if not body:
 932         if tail_len == 0:
 933             raise CannotSplit("Splitting brackets produced the same line")
 934
 935         elif tail_len < 3:
 936             raise CannotSplit(
 937                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 938                 " not worth it"
 939             )
 940
 941
 942 def bracket_split_build_line(
 943     leaves: List[Leaf],
 944     original: Line,
 945     opening_bracket: Leaf,
 946     *,
 947     component: _BracketSplitComponent,
 948 ) -> Line:
 949     """Return a new line with given `leaves` and respective comments from `original`.
 950
 951     If it's the head component, brackets will be tracked so trailing commas are
 952     respected.
 953
 954     If it's the body component, the result line is one-indented inside brackets and as
 955     such has its first leaf's prefix normalized and a trailing comma added when
 956     expected.
 957     """
 958     result = Line(mode=original.mode, depth=original.depth)
 959     if component is _BracketSplitComponent.body:
 960         result.inside_brackets = True
 961         result.depth += 1
 962         if leaves:
 963             # Since body is a new indent level, remove spurious leading whitespace.
 964             normalize_prefix(leaves[0], inside_brackets=True)
 965             # Ensure a trailing comma for imports and standalone function arguments, but
 966             # be careful not to add one after any comments or within type annotations.
 967             no_commas = (
 968                 original.is_def
 969                 and opening_bracket.value == "("
 970                 and not any(leaf.type == token.COMMA for leaf in leaves)
 971                 # In particular, don't add one within a parenthesized return annotation.
 972                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 973                 # be defined directly in the parent node, the parent of the parent ...
 974                 # and so on depending on how complex the return annotation is.
 975                 # This isn't perfect and there's some false negatives but they are in
 976                 # contexts were a comma is actually fine.
 977                 and not any(
 978                     node.prev_sibling.type == RARROW
 979                     for node in (
 980                         leaves[0].parent,
 981                         getattr(leaves[0].parent, "parent", None),
 982                     )
 983                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 984                 )
 985                 # Except the false negatives above for PEP 604 unions where we
 986                 # can't add the comma.
 987                 and not (
 988                     leaves[0].parent
 989                     and leaves[0].parent.next_sibling
 990                     and leaves[0].parent.next_sibling.type == token.VBAR
 991                 )
 992             )
 993
 994             if original.is_import or no_commas:
 995                 for i in range(len(leaves) - 1, -1, -1):
 996                     if leaves[i].type == STANDALONE_COMMENT:
 997                         continue
 998
 999                     if leaves[i].type != token.COMMA:
1000                         new_comma = Leaf(token.COMMA, ",")
1001                         leaves.insert(i + 1, new_comma)
1002                     break
1003
1004     leaves_to_track: Set[LeafID] = set()
1005     if component is _BracketSplitComponent.head:
1006         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1007     # Populate the line
1008     for leaf in leaves:
1009         result.append(
1010             leaf,
1011             preformatted=True,
1012             track_bracket=id(leaf) in leaves_to_track,
1013         )
1014         for comment_after in original.comments_after(leaf):
1015             result.append(comment_after, preformatted=True)
1016     if component is _BracketSplitComponent.body and should_split_line(
1017         result, opening_bracket
1018     ):
1019         result.should_split_rhs = True
1020     return result
1021
1022
1023 def dont_increase_indentation(split_func: Transformer) -> Transformer:
1024     """Normalize prefix of the first leaf in every line returned by `split_func`.
1025
1026     This is a decorator over relevant split functions.
1027     """
1028
1029     @wraps(split_func)
1030     def split_wrapper(
1031         line: Line, features: Collection[Feature], mode: Mode
1032     ) -> Iterator[Line]:
1033         for split_line in split_func(line, features, mode):
1034             normalize_prefix(split_line.leaves[0], inside_brackets=True)
1035             yield split_line
1036
1037     return split_wrapper
1038
1039
1040 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1041     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1042         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1043             return leaf_idx
1044     return None
1045
1046
1047 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1048     if (
1049         safe
1050         and delimiter_priority == COMMA_PRIORITY
1051         and line.leaves[-1].type != token.COMMA
1052         and line.leaves[-1].type != STANDALONE_COMMENT
1053     ):
1054         new_comma = Leaf(token.COMMA, ",")
1055         line.append(new_comma)
1056     return line
1057
1058
1059 @dont_increase_indentation
1060 def delimiter_split(
1061     line: Line, features: Collection[Feature], mode: Mode
1062 ) -> Iterator[Line]:
1063     """Split according to delimiters of the highest priority.
1064
1065     If the appropriate Features are given, the split will add trailing commas
1066     also in function signatures and calls that contain `*` and `**`.
1067     """
1068     try:
1069         last_leaf = line.leaves[-1]
1070     except IndexError:
1071         raise CannotSplit("Line empty") from None
1072
1073     bt = line.bracket_tracker
1074     try:
1075         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1076     except ValueError:
1077         raise CannotSplit("No delimiters found") from None
1078
1079     if delimiter_priority == DOT_PRIORITY:
1080         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
1081             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1082
1083     current_line = Line(
1084         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1085     )
1086     lowest_depth = sys.maxsize
1087     trailing_comma_safe = True
1088
1089     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1090         """Append `leaf` to current line or to new line if appending impossible."""
1091         nonlocal current_line
1092         try:
1093             current_line.append_safe(leaf, preformatted=True)
1094         except ValueError:
1095             yield current_line
1096
1097             current_line = Line(
1098                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1099             )
1100             current_line.append(leaf)
1101
1102     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1103     for leaf_idx, leaf in enumerate(line.leaves):
1104         yield from append_to_line(leaf)
1105
1106         for comment_after in line.comments_after(leaf):
1107             yield from append_to_line(comment_after)
1108
1109         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1110         if leaf.bracket_depth == lowest_depth:
1111             if is_vararg(leaf, within={syms.typedargslist}):
1112                 trailing_comma_safe = (
1113                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1114                 )
1115             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1116                 trailing_comma_safe = (
1117                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1118                 )
1119
1120         if (
1121             Preview.add_trailing_comma_consistently in mode
1122             and last_leaf.type == STANDALONE_COMMENT
1123             and leaf_idx == last_non_comment_leaf
1124         ):
1125             current_line = _safe_add_trailing_comma(
1126                 trailing_comma_safe, delimiter_priority, current_line
1127             )
1128
1129         leaf_priority = bt.delimiters.get(id(leaf))
1130         if leaf_priority == delimiter_priority:
1131             yield current_line
1132
1133             current_line = Line(
1134                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1135             )
1136     if current_line:
1137         current_line = _safe_add_trailing_comma(
1138             trailing_comma_safe, delimiter_priority, current_line
1139         )
1140         yield current_line
1141
1142
1143 @dont_increase_indentation
1144 def standalone_comment_split(
1145     line: Line, features: Collection[Feature], mode: Mode
1146 ) -> Iterator[Line]:
1147     """Split standalone comments from the rest of the line."""
1148     if not line.contains_standalone_comments(0):
1149         raise CannotSplit("Line does not have any standalone comments")
1150
1151     current_line = Line(
1152         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1153     )
1154
1155     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1156         """Append `leaf` to current line or to new line if appending impossible."""
1157         nonlocal current_line
1158         try:
1159             current_line.append_safe(leaf, preformatted=True)
1160         except ValueError:
1161             yield current_line
1162
1163             current_line = Line(
1164                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1165             )
1166             current_line.append(leaf)
1167
1168     for leaf in line.leaves:
1169         yield from append_to_line(leaf)
1170
1171         for comment_after in line.comments_after(leaf):
1172             yield from append_to_line(comment_after)
1173
1174     if current_line:
1175         yield current_line
1176
1177
1178 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1179     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1180     else.
1181
1182     Note: don't use backslashes for formatting or you'll lose your voting rights.
1183     """
1184     if not inside_brackets:
1185         spl = leaf.prefix.split("#")
1186         if "\\" not in spl[0]:
1187             nl_count = spl[-1].count("\n")
1188             if len(spl) > 1:
1189                 nl_count -= 1
1190             leaf.prefix = "\n" * nl_count
1191             return
1192
1193     leaf.prefix = ""
1194
1195
1196 def normalize_invisible_parens(
1197     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1198 ) -> None:
1199     """Make existing optional parentheses invisible or create new ones.
1200
1201     `parens_after` is a set of string leaf values immediately after which parens
1202     should be put.
1203
1204     Standardizes on visible parentheses for single-element tuples, and keeps
1205     existing visible parentheses for other tuples and generator expressions.
1206     """
1207     for pc in list_comments(node.prefix, is_endmarker=False):
1208         if pc.value in FMT_OFF:
1209             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1210             return
1211
1212     # The multiple context managers grammar has a different pattern, thus this is
1213     # separate from the for-loop below. This possibly wraps them in invisible parens,
1214     # and later will be removed in remove_with_parens when needed.
1215     if node.type == syms.with_stmt:
1216         _maybe_wrap_cms_in_parens(node, mode, features)
1217
1218     check_lpar = False
1219     for index, child in enumerate(list(node.children)):
1220         # Fixes a bug where invisible parens are not properly stripped from
1221         # assignment statements that contain type annotations.
1222         if isinstance(child, Node) and child.type == syms.annassign:
1223             normalize_invisible_parens(
1224                 child, parens_after=parens_after, mode=mode, features=features
1225             )
1226
1227         # Add parentheses around long tuple unpacking in assignments.
1228         if (
1229             index == 0
1230             and isinstance(child, Node)
1231             and child.type == syms.testlist_star_expr
1232         ):
1233             check_lpar = True
1234
1235         if check_lpar:
1236             if (
1237                 child.type == syms.atom
1238                 and node.type == syms.for_stmt
1239                 and isinstance(child.prev_sibling, Leaf)
1240                 and child.prev_sibling.type == token.NAME
1241                 and child.prev_sibling.value == "for"
1242             ):
1243                 if maybe_make_parens_invisible_in_atom(
1244                     child,
1245                     parent=node,
1246                     remove_brackets_around_comma=True,
1247                 ):
1248                     wrap_in_parentheses(node, child, visible=False)
1249             elif isinstance(child, Node) and node.type == syms.with_stmt:
1250                 remove_with_parens(child, node)
1251             elif child.type == syms.atom:
1252                 if maybe_make_parens_invisible_in_atom(
1253                     child,
1254                     parent=node,
1255                 ):
1256                     wrap_in_parentheses(node, child, visible=False)
1257             elif is_one_tuple(child):
1258                 wrap_in_parentheses(node, child, visible=True)
1259             elif node.type == syms.import_from:
1260                 _normalize_import_from(node, child, index)
1261                 break
1262             elif (
1263                 index == 1
1264                 and child.type == token.STAR
1265                 and node.type == syms.except_clause
1266             ):
1267                 # In except* (PEP 654), the star is actually part of
1268                 # of the keyword. So we need to skip the insertion of
1269                 # invisible parentheses to work more precisely.
1270                 continue
1271
1272             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1273                 wrap_in_parentheses(node, child, visible=False)
1274
1275         comma_check = child.type == token.COMMA
1276
1277         check_lpar = isinstance(child, Leaf) and (
1278             child.value in parens_after or comma_check
1279         )
1280
1281
1282 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1283     # "import from" nodes store parentheses directly as part of
1284     # the statement
1285     if is_lpar_token(child):
1286         assert is_rpar_token(parent.children[-1])
1287         # make parentheses invisible
1288         child.value = ""
1289         parent.children[-1].value = ""
1290     elif child.type != token.STAR:
1291         # insert invisible parentheses
1292         parent.insert_child(index, Leaf(token.LPAR, ""))
1293         parent.append_child(Leaf(token.RPAR, ""))
1294
1295
1296 def remove_await_parens(node: Node) -> None:
1297     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1298         if (
1299             node.children[1].type == syms.atom
1300             and node.children[1].children[0].type == token.LPAR
1301         ):
1302             if maybe_make_parens_invisible_in_atom(
1303                 node.children[1],
1304                 parent=node,
1305                 remove_brackets_around_comma=True,
1306             ):
1307                 wrap_in_parentheses(node, node.children[1], visible=False)
1308
1309             # Since await is an expression we shouldn't remove
1310             # brackets in cases where this would change
1311             # the AST due to operator precedence.
1312             # Therefore we only aim to remove brackets around
1313             # power nodes that aren't also await expressions themselves.
1314             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1315             # N.B. We've still removed any redundant nested brackets though :)
1316             opening_bracket = cast(Leaf, node.children[1].children[0])
1317             closing_bracket = cast(Leaf, node.children[1].children[-1])
1318             bracket_contents = node.children[1].children[1]
1319             if isinstance(bracket_contents, Node):
1320                 if bracket_contents.type != syms.power:
1321                     ensure_visible(opening_bracket)
1322                     ensure_visible(closing_bracket)
1323                 elif (
1324                     bracket_contents.type == syms.power
1325                     and bracket_contents.children[0].type == token.AWAIT
1326                 ):
1327                     ensure_visible(opening_bracket)
1328                     ensure_visible(closing_bracket)
1329                     # If we are in a nested await then recurse down.
1330                     remove_await_parens(bracket_contents)
1331
1332
1333 def _maybe_wrap_cms_in_parens(
1334     node: Node, mode: Mode, features: Collection[Feature]
1335 ) -> None:
1336     """When enabled and safe, wrap the multiple context managers in invisible parens.
1337
1338     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1339     """
1340     if (
1341         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1342         or Preview.wrap_multiple_context_managers_in_parens not in mode
1343         or len(node.children) <= 2
1344         # If it's an atom, it's already wrapped in parens.
1345         or node.children[1].type == syms.atom
1346     ):
1347         return
1348     colon_index: Optional[int] = None
1349     for i in range(2, len(node.children)):
1350         if node.children[i].type == token.COLON:
1351             colon_index = i
1352             break
1353     if colon_index is not None:
1354         lpar = Leaf(token.LPAR, "")
1355         rpar = Leaf(token.RPAR, "")
1356         context_managers = node.children[1:colon_index]
1357         for child in context_managers:
1358             child.remove()
1359         # After wrapping, the with_stmt will look like this:
1360         #   with_stmt
1361         #     NAME 'with'
1362         #     atom
1363         #       LPAR ''
1364         #       testlist_gexp
1365         #         ... <-- context_managers
1366         #       /testlist_gexp
1367         #       RPAR ''
1368         #     /atom
1369         #     COLON ':'
1370         new_child = Node(
1371             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1372         )
1373         node.insert_child(1, new_child)
1374
1375
1376 def remove_with_parens(node: Node, parent: Node) -> None:
1377     """Recursively hide optional parens in `with` statements."""
1378     # Removing all unnecessary parentheses in with statements in one pass is a tad
1379     # complex as different variations of bracketed statements result in pretty
1380     # different parse trees:
1381     #
1382     # with (open("file")) as f:                       # this is an asexpr_test
1383     #     ...
1384     #
1385     # with (open("file") as f):                       # this is an atom containing an
1386     #     ...                                         # asexpr_test
1387     #
1388     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1389     #     ...                                         # asexpr_test
1390     #
1391     # with (open("file") as f, open("file") as f):    # an atom containing a
1392     #     ...                                         # testlist_gexp which then
1393     #                                                 # contains multiple asexpr_test(s)
1394     if node.type == syms.atom:
1395         if maybe_make_parens_invisible_in_atom(
1396             node,
1397             parent=parent,
1398             remove_brackets_around_comma=True,
1399         ):
1400             wrap_in_parentheses(parent, node, visible=False)
1401         if isinstance(node.children[1], Node):
1402             remove_with_parens(node.children[1], node)
1403     elif node.type == syms.testlist_gexp:
1404         for child in node.children:
1405             if isinstance(child, Node):
1406                 remove_with_parens(child, node)
1407     elif node.type == syms.asexpr_test and not any(
1408         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1409     ):
1410         if maybe_make_parens_invisible_in_atom(
1411             node.children[0],
1412             parent=node,
1413             remove_brackets_around_comma=True,
1414         ):
1415             wrap_in_parentheses(node, node.children[0], visible=False)
1416
1417
1418 def maybe_make_parens_invisible_in_atom(
1419     node: LN,
1420     parent: LN,
1421     remove_brackets_around_comma: bool = False,
1422 ) -> bool:
1423     """If it's safe, make the parens in the atom `node` invisible, recursively.
1424     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1425     as they are redundant.
1426
1427     Returns whether the node should itself be wrapped in invisible parentheses.
1428     """
1429     if (
1430         node.type not in (syms.atom, syms.expr)
1431         or is_empty_tuple(node)
1432         or is_one_tuple(node)
1433         or (is_yield(node) and parent.type != syms.expr_stmt)
1434         or (
1435             # This condition tries to prevent removing non-optional brackets
1436             # around a tuple, however, can be a bit overzealous so we provide
1437             # and option to skip this check for `for` and `with` statements.
1438             not remove_brackets_around_comma
1439             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1440         )
1441         or is_tuple_containing_walrus(node)
1442     ):
1443         return False
1444
1445     if is_walrus_assignment(node):
1446         if parent.type in [
1447             syms.annassign,
1448             syms.expr_stmt,
1449             syms.assert_stmt,
1450             syms.return_stmt,
1451             syms.except_clause,
1452             syms.funcdef,
1453             syms.with_stmt,
1454             syms.tname,
1455             # these ones aren't useful to end users, but they do please fuzzers
1456             syms.for_stmt,
1457             syms.del_stmt,
1458             syms.for_stmt,
1459         ]:
1460             return False
1461
1462     first = node.children[0]
1463     last = node.children[-1]
1464     if is_lpar_token(first) and is_rpar_token(last):
1465         middle = node.children[1]
1466         # make parentheses invisible
1467         if (
1468             # If the prefix of `middle` includes a type comment with
1469             # ignore annotation, then we do not remove the parentheses
1470             not is_type_ignore_comment_string(middle.prefix.strip())
1471         ):
1472             first.value = ""
1473             last.value = ""
1474         maybe_make_parens_invisible_in_atom(
1475             middle,
1476             parent=parent,
1477             remove_brackets_around_comma=remove_brackets_around_comma,
1478         )
1479
1480         if is_atom_with_invisible_parens(middle):
1481             # Strip the invisible parens from `middle` by replacing
1482             # it with the child in-between the invisible parens
1483             middle.replace(middle.children[1])
1484
1485         return False
1486
1487     return True
1488
1489
1490 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1491     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1492
1493     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1494         return False
1495
1496     # We're essentially checking if the body is delimited by commas and there's more
1497     # than one of them (we're excluding the trailing comma and if the delimiter priority
1498     # is still commas, that means there's more).
1499     exclude = set()
1500     trailing_comma = False
1501     try:
1502         last_leaf = line.leaves[-1]
1503         if last_leaf.type == token.COMMA:
1504             trailing_comma = True
1505             exclude.add(id(last_leaf))
1506         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1507     except (IndexError, ValueError):
1508         return False
1509
1510     return max_priority == COMMA_PRIORITY and (
1511         (line.mode.magic_trailing_comma and trailing_comma)
1512         # always explode imports
1513         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1514     )
1515
1516
1517 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1518     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1519
1520     Brackets can be omitted if the entire trailer up to and including
1521     a preceding closing bracket fits in one line.
1522
1523     Yielded sets are cumulative (contain results of previous yields, too).  First
1524     set is empty, unless the line should explode, in which case bracket pairs until
1525     the one that needs to explode are omitted.
1526     """
1527
1528     omit: Set[LeafID] = set()
1529     if not line.magic_trailing_comma:
1530         yield omit
1531
1532     length = 4 * line.depth
1533     opening_bracket: Optional[Leaf] = None
1534     closing_bracket: Optional[Leaf] = None
1535     inner_brackets: Set[LeafID] = set()
1536     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1537         length += leaf_length
1538         if length > line_length:
1539             break
1540
1541         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1542         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1543             break
1544
1545         if opening_bracket:
1546             if leaf is opening_bracket:
1547                 opening_bracket = None
1548             elif leaf.type in CLOSING_BRACKETS:
1549                 prev = line.leaves[index - 1] if index > 0 else None
1550                 if (
1551                     prev
1552                     and prev.type == token.COMMA
1553                     and leaf.opening_bracket is not None
1554                     and not is_one_sequence_between(
1555                         leaf.opening_bracket, leaf, line.leaves
1556                     )
1557                 ):
1558                     # Never omit bracket pairs with trailing commas.
1559                     # We need to explode on those.
1560                     break
1561
1562                 inner_brackets.add(id(leaf))
1563         elif leaf.type in CLOSING_BRACKETS:
1564             prev = line.leaves[index - 1] if index > 0 else None
1565             if prev and prev.type in OPENING_BRACKETS:
1566                 # Empty brackets would fail a split so treat them as "inner"
1567                 # brackets (e.g. only add them to the `omit` set if another
1568                 # pair of brackets was good enough.
1569                 inner_brackets.add(id(leaf))
1570                 continue
1571
1572             if closing_bracket:
1573                 omit.add(id(closing_bracket))
1574                 omit.update(inner_brackets)
1575                 inner_brackets.clear()
1576                 yield omit
1577
1578             if (
1579                 prev
1580                 and prev.type == token.COMMA
1581                 and leaf.opening_bracket is not None
1582                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1583             ):
1584                 # Never omit bracket pairs with trailing commas.
1585                 # We need to explode on those.
1586                 break
1587
1588             if leaf.value:
1589                 opening_bracket = leaf.opening_bracket
1590                 closing_bracket = leaf
1591
1592
1593 def run_transformer(
1594     line: Line,
1595     transform: Transformer,
1596     mode: Mode,
1597     features: Collection[Feature],
1598     *,
1599     line_str: str = "",
1600 ) -> List[Line]:
1601     if not line_str:
1602         line_str = line_to_string(line)
1603     result: List[Line] = []
1604     for transformed_line in transform(line, features, mode):
1605         if str(transformed_line).strip("\n") == line_str:
1606             raise CannotTransform("Line transformer returned an unchanged result")
1607
1608         result.extend(transform_line(transformed_line, mode=mode, features=features))
1609
1610     features_set = set(features)
1611     if (
1612         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1613         or transform.__class__.__name__ != "rhs"
1614         or not line.bracket_tracker.invisible
1615         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1616         or line.contains_multiline_strings()
1617         or result[0].contains_uncollapsable_type_comments()
1618         or result[0].contains_unsplittable_type_ignore()
1619         or is_line_short_enough(result[0], mode=mode)
1620         # If any leaves have no parents (which _can_ occur since
1621         # `transform(line)` potentially destroys the line's underlying node
1622         # structure), then we can't proceed. Doing so would cause the below
1623         # call to `append_leaves()` to fail.
1624         or any(leaf.parent is None for leaf in line.leaves)
1625     ):
1626         return result
1627
1628     line_copy = line.clone()
1629     append_leaves(line_copy, line, line.leaves)
1630     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1631     second_opinion = run_transformer(
1632         line_copy, transform, mode, features_fop, line_str=line_str
1633     )
1634     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1635         result = second_opinion
1636     return result