src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import replace
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     RHSResult,
  20     append_leaves,
  21     can_be_split,
  22     can_omit_invisible_parens,
  23     is_line_short_enough,
  24     line_to_string,
  25 )
  26 from black.mode import Feature, Mode, Preview
  27 from black.nodes import (
  28     ASSIGNMENTS,
  29     BRACKETS,
  30     CLOSING_BRACKETS,
  31     OPENING_BRACKETS,
  32     RARROW,
  33     STANDALONE_COMMENT,
  34     STATEMENT,
  35     WHITESPACE,
  36     Visitor,
  37     ensure_visible,
  38     is_arith_like,
  39     is_async_stmt_or_funcdef,
  40     is_atom_with_invisible_parens,
  41     is_docstring,
  42     is_empty_tuple,
  43     is_lpar_token,
  44     is_multiline_string,
  45     is_name_token,
  46     is_one_sequence_between,
  47     is_one_tuple,
  48     is_rpar_token,
  49     is_stub_body,
  50     is_stub_suite,
  51     is_tuple_containing_walrus,
  52     is_type_ignore_comment_string,
  53     is_vararg,
  54     is_walrus_assignment,
  55     is_yield,
  56     syms,
  57     wrap_in_parentheses,
  58 )
  59 from black.numerics import normalize_numeric_literal
  60 from black.strings import (
  61     fix_docstring,
  62     get_string_prefix,
  63     normalize_string_prefix,
  64     normalize_string_quotes,
  65     normalize_unicode_escape_sequences,
  66 )
  67 from black.trans import (
  68     CannotTransform,
  69     StringMerger,
  70     StringParenStripper,
  71     StringParenWrapper,
  72     StringSplitter,
  73     Transformer,
  74     hug_power_op,
  75 )
  76 from blib2to3.pgen2 import token
  77 from blib2to3.pytree import Leaf, Node
  78
  79 # types
  80 LeafID = int
  81 LN = Union[Leaf, Node]
  82
  83
  84 class CannotSplit(CannotTransform):
  85     """A readable split that fits the allotted line length is impossible."""
  86
  87
  88 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  89 # See also https://github.com/mypyc/mypyc/issues/827.
  90 class LineGenerator(Visitor[Line]):
  91     """Generates reformatted Line objects.  Empty lines are not emitted.
  92
  93     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  94     in ways that will no longer stringify to valid Python code on the tree.
  95     """
  96
  97     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  98         self.mode = mode
  99         self.features = features
 100         self.current_line: Line
 101         self.__post_init__()
 102
 103     def line(self, indent: int = 0) -> Iterator[Line]:
 104         """Generate a line.
 105
 106         If the line is empty, only emit if it makes sense.
 107         If the line is too long, split it first and then generate.
 108
 109         If any lines were generated, set up a new current_line.
 110         """
 111         if not self.current_line:
 112             self.current_line.depth += indent
 113             return  # Line is empty, don't emit. Creating a new one unnecessary.
 114
 115         if (
 116             Preview.improved_async_statements_handling in self.mode
 117             and len(self.current_line.leaves) == 1
 118             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
 119         ):
 120             # Special case for async def/for/with statements. `visit_async_stmt`
 121             # adds an `ASYNC` leaf then visits the child def/for/with statement
 122             # nodes. Line yields from those nodes shouldn't treat the former
 123             # `ASYNC` leaf as a complete line.
 124             return
 125
 126         complete_line = self.current_line
 127         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 128         yield complete_line
 129
 130     def visit_default(self, node: LN) -> Iterator[Line]:
 131         """Default `visit_*()` implementation. Recurses to children of `node`."""
 132         if isinstance(node, Leaf):
 133             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 134             for comment in generate_comments(node):
 135                 if any_open_brackets:
 136                     # any comment within brackets is subject to splitting
 137                     self.current_line.append(comment)
 138                 elif comment.type == token.COMMENT:
 139                     # regular trailing comment
 140                     self.current_line.append(comment)
 141                     yield from self.line()
 142
 143                 else:
 144                     # regular standalone comment
 145                     yield from self.line()
 146
 147                     self.current_line.append(comment)
 148                     yield from self.line()
 149
 150             normalize_prefix(node, inside_brackets=any_open_brackets)
 151             if self.mode.string_normalization and node.type == token.STRING:
 152                 node.value = normalize_string_prefix(node.value)
 153                 node.value = normalize_string_quotes(node.value)
 154             if node.type == token.NUMBER:
 155                 normalize_numeric_literal(node)
 156             if node.type not in WHITESPACE:
 157                 self.current_line.append(node)
 158         yield from super().visit_default(node)
 159
 160     def visit_test(self, node: Node) -> Iterator[Line]:
 161         """Visit an `x if y else z` test"""
 162
 163         if Preview.parenthesize_conditional_expressions in self.mode:
 164             already_parenthesized = (
 165                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 166             )
 167
 168             if not already_parenthesized:
 169                 lpar = Leaf(token.LPAR, "")
 170                 rpar = Leaf(token.RPAR, "")
 171                 node.insert_child(0, lpar)
 172                 node.append_child(rpar)
 173
 174         yield from self.visit_default(node)
 175
 176     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 177         """Increase indentation level, maybe yield a line."""
 178         # In blib2to3 INDENT never holds comments.
 179         yield from self.line(+1)
 180         yield from self.visit_default(node)
 181
 182     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 183         """Decrease indentation level, maybe yield a line."""
 184         # The current line might still wait for trailing comments.  At DEDENT time
 185         # there won't be any (they would be prefixes on the preceding NEWLINE).
 186         # Emit the line then.
 187         yield from self.line()
 188
 189         # While DEDENT has no value, its prefix may contain standalone comments
 190         # that belong to the current indentation level.  Get 'em.
 191         yield from self.visit_default(node)
 192
 193         # Finally, emit the dedent.
 194         yield from self.line(-1)
 195
 196     def visit_stmt(
 197         self, node: Node, keywords: Set[str], parens: Set[str]
 198     ) -> Iterator[Line]:
 199         """Visit a statement.
 200
 201         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 202         `def`, `with`, `class`, `assert`, and assignments.
 203
 204         The relevant Python language `keywords` for a given statement will be
 205         NAME leaves within it. This methods puts those on a separate line.
 206
 207         `parens` holds a set of string leaf values immediately after which
 208         invisible parens should be put.
 209         """
 210         normalize_invisible_parens(
 211             node, parens_after=parens, mode=self.mode, features=self.features
 212         )
 213         for child in node.children:
 214             if is_name_token(child) and child.value in keywords:
 215                 yield from self.line()
 216
 217             yield from self.visit(child)
 218
 219     def visit_typeparams(self, node: Node) -> Iterator[Line]:
 220         yield from self.visit_default(node)
 221         node.children[0].prefix = ""
 222
 223     def visit_typevartuple(self, node: Node) -> Iterator[Line]:
 224         yield from self.visit_default(node)
 225         node.children[1].prefix = ""
 226
 227     def visit_paramspec(self, node: Node) -> Iterator[Line]:
 228         yield from self.visit_default(node)
 229         node.children[1].prefix = ""
 230
 231     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 232         if Preview.wrap_long_dict_values_in_parens in self.mode:
 233             for i, child in enumerate(node.children):
 234                 if i == 0:
 235                     continue
 236                 if node.children[i - 1].type == token.COLON:
 237                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 238                         if maybe_make_parens_invisible_in_atom(
 239                             child,
 240                             parent=node,
 241                             remove_brackets_around_comma=False,
 242                         ):
 243                             wrap_in_parentheses(node, child, visible=False)
 244                     else:
 245                         wrap_in_parentheses(node, child, visible=False)
 246         yield from self.visit_default(node)
 247
 248     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 249         """Visit function definition."""
 250         yield from self.line()
 251
 252         # Remove redundant brackets around return type annotation.
 253         is_return_annotation = False
 254         for child in node.children:
 255             if child.type == token.RARROW:
 256                 is_return_annotation = True
 257             elif is_return_annotation:
 258                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 259                     if maybe_make_parens_invisible_in_atom(
 260                         child,
 261                         parent=node,
 262                         remove_brackets_around_comma=False,
 263                     ):
 264                         wrap_in_parentheses(node, child, visible=False)
 265                 else:
 266                     wrap_in_parentheses(node, child, visible=False)
 267                 is_return_annotation = False
 268
 269         for child in node.children:
 270             yield from self.visit(child)
 271
 272     def visit_match_case(self, node: Node) -> Iterator[Line]:
 273         """Visit either a match or case statement."""
 274         normalize_invisible_parens(
 275             node, parens_after=set(), mode=self.mode, features=self.features
 276         )
 277
 278         yield from self.line()
 279         for child in node.children:
 280             yield from self.visit(child)
 281
 282     def visit_suite(self, node: Node) -> Iterator[Line]:
 283         """Visit a suite."""
 284         if self.mode.is_pyi and is_stub_suite(node):
 285             yield from self.visit(node.children[2])
 286         else:
 287             yield from self.visit_default(node)
 288
 289     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 290         """Visit a statement without nested statements."""
 291         prev_type: Optional[int] = None
 292         for child in node.children:
 293             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 294                 wrap_in_parentheses(node, child, visible=False)
 295             prev_type = child.type
 296
 297         is_suite_like = node.parent and node.parent.type in STATEMENT
 298         if is_suite_like:
 299             if self.mode.is_pyi and is_stub_body(node):
 300                 yield from self.visit_default(node)
 301             else:
 302                 yield from self.line(+1)
 303                 yield from self.visit_default(node)
 304                 yield from self.line(-1)
 305
 306         else:
 307             if (
 308                 not self.mode.is_pyi
 309                 or not node.parent
 310                 or not is_stub_suite(node.parent)
 311             ):
 312                 yield from self.line()
 313             yield from self.visit_default(node)
 314
 315     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 316         """Visit `async def`, `async for`, `async with`."""
 317         yield from self.line()
 318
 319         children = iter(node.children)
 320         for child in children:
 321             yield from self.visit(child)
 322
 323             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 324                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 325                 # line.
 326                 break
 327
 328         internal_stmt = next(children)
 329         if Preview.improved_async_statements_handling in self.mode:
 330             yield from self.visit(internal_stmt)
 331         else:
 332             for child in internal_stmt.children:
 333                 yield from self.visit(child)
 334
 335     def visit_decorators(self, node: Node) -> Iterator[Line]:
 336         """Visit decorators."""
 337         for child in node.children:
 338             yield from self.line()
 339             yield from self.visit(child)
 340
 341     def visit_power(self, node: Node) -> Iterator[Line]:
 342         for idx, leaf in enumerate(node.children[:-1]):
 343             next_leaf = node.children[idx + 1]
 344
 345             if not isinstance(leaf, Leaf):
 346                 continue
 347
 348             value = leaf.value.lower()
 349             if (
 350                 leaf.type == token.NUMBER
 351                 and next_leaf.type == syms.trailer
 352                 # Ensure that we are in an attribute trailer
 353                 and next_leaf.children[0].type == token.DOT
 354                 # It shouldn't wrap hexadecimal, binary and octal literals
 355                 and not value.startswith(("0x", "0b", "0o"))
 356                 # It shouldn't wrap complex literals
 357                 and "j" not in value
 358             ):
 359                 wrap_in_parentheses(node, leaf)
 360
 361         remove_await_parens(node)
 362
 363         yield from self.visit_default(node)
 364
 365     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 366         """Remove a semicolon and put the other statement on a separate line."""
 367         yield from self.line()
 368
 369     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 370         """End of file. Process outstanding comments and end with a newline."""
 371         yield from self.visit_default(leaf)
 372         yield from self.line()
 373
 374     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 375         if not self.current_line.bracket_tracker.any_open_brackets():
 376             yield from self.line()
 377         yield from self.visit_default(leaf)
 378
 379     def visit_factor(self, node: Node) -> Iterator[Line]:
 380         """Force parentheses between a unary op and a binary power:
 381
 382         -2 ** 8 -> -(2 ** 8)
 383         """
 384         _operator, operand = node.children
 385         if (
 386             operand.type == syms.power
 387             and len(operand.children) == 3
 388             and operand.children[1].type == token.DOUBLESTAR
 389         ):
 390             lpar = Leaf(token.LPAR, "(")
 391             rpar = Leaf(token.RPAR, ")")
 392             index = operand.remove() or 0
 393             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 394         yield from self.visit_default(node)
 395
 396     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 397         if Preview.hex_codes_in_unicode_sequences in self.mode:
 398             normalize_unicode_escape_sequences(leaf)
 399
 400         if is_docstring(leaf) and "\\\n" not in leaf.value:
 401             # We're ignoring docstrings with backslash newline escapes because changing
 402             # indentation of those changes the AST representation of the code.
 403             if self.mode.string_normalization:
 404                 docstring = normalize_string_prefix(leaf.value)
 405                 # visit_default() does handle string normalization for us, but
 406                 # since this method acts differently depending on quote style (ex.
 407                 # see padding logic below), there's a possibility for unstable
 408                 # formatting as visit_default() is called *after*. To avoid a
 409                 # situation where this function formats a docstring differently on
 410                 # the second pass, normalize it early.
 411                 docstring = normalize_string_quotes(docstring)
 412             else:
 413                 docstring = leaf.value
 414             prefix = get_string_prefix(docstring)
 415             docstring = docstring[len(prefix) :]  # Remove the prefix
 416             quote_char = docstring[0]
 417             # A natural way to remove the outer quotes is to do:
 418             #   docstring = docstring.strip(quote_char)
 419             # but that breaks on """""x""" (which is '""x').
 420             # So we actually need to remove the first character and the next two
 421             # characters but only if they are the same as the first.
 422             quote_len = 1 if docstring[1] != quote_char else 3
 423             docstring = docstring[quote_len:-quote_len]
 424             docstring_started_empty = not docstring
 425             indent = " " * 4 * self.current_line.depth
 426
 427             if is_multiline_string(leaf):
 428                 docstring = fix_docstring(docstring, indent)
 429             else:
 430                 docstring = docstring.strip()
 431
 432             has_trailing_backslash = False
 433             if docstring:
 434                 # Add some padding if the docstring starts / ends with a quote mark.
 435                 if docstring[0] == quote_char:
 436                     docstring = " " + docstring
 437                 if docstring[-1] == quote_char:
 438                     docstring += " "
 439                 if docstring[-1] == "\\":
 440                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 441                     if backslash_count % 2:
 442                         # Odd number of tailing backslashes, add some padding to
 443                         # avoid escaping the closing string quote.
 444                         docstring += " "
 445                         has_trailing_backslash = True
 446             elif not docstring_started_empty:
 447                 docstring = " "
 448
 449             # We could enforce triple quotes at this point.
 450             quote = quote_char * quote_len
 451
 452             # It's invalid to put closing single-character quotes on a new line.
 453             if self.mode and quote_len == 3:
 454                 # We need to find the length of the last line of the docstring
 455                 # to find if we can add the closing quotes to the line without
 456                 # exceeding the maximum line length.
 457                 # If docstring is one line, we don't put the closing quotes on a
 458                 # separate line because it looks ugly (#3320).
 459                 lines = docstring.splitlines()
 460                 last_line_length = len(lines[-1]) if docstring else 0
 461
 462                 # If adding closing quotes would cause the last line to exceed
 463                 # the maximum line length then put a line break before the
 464                 # closing quotes
 465                 if (
 466                     len(lines) > 1
 467                     and last_line_length + quote_len > self.mode.line_length
 468                     and len(indent) + quote_len <= self.mode.line_length
 469                     and not has_trailing_backslash
 470                 ):
 471                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 472                 else:
 473                     leaf.value = prefix + quote + docstring + quote
 474             else:
 475                 leaf.value = prefix + quote + docstring + quote
 476
 477         yield from self.visit_default(leaf)
 478
 479     def __post_init__(self) -> None:
 480         """You are in a twisty little maze of passages."""
 481         self.current_line = Line(mode=self.mode)
 482
 483         v = self.visit_stmt
 484         Ø: Set[str] = set()
 485         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 486         self.visit_if_stmt = partial(
 487             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 488         )
 489         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 490         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 491         self.visit_try_stmt = partial(
 492             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 493         )
 494         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 495         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 496         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 497         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 498         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 499         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 500         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 501         self.visit_async_funcdef = self.visit_async_stmt
 502         self.visit_decorated = self.visit_decorators
 503
 504         # PEP 634
 505         self.visit_match_stmt = self.visit_match_case
 506         self.visit_case_block = self.visit_match_case
 507
 508
 509 def transform_line(
 510     line: Line, mode: Mode, features: Collection[Feature] = ()
 511 ) -> Iterator[Line]:
 512     """Transform a `line`, potentially splitting it into many lines.
 513
 514     They should fit in the allotted `line_length` but might not be able to.
 515
 516     `features` are syntactical features that may be used in the output.
 517     """
 518     if line.is_comment:
 519         yield line
 520         return
 521
 522     line_str = line_to_string(line)
 523
 524     ll = mode.line_length
 525     sn = mode.string_normalization
 526     string_merge = StringMerger(ll, sn)
 527     string_paren_strip = StringParenStripper(ll, sn)
 528     string_split = StringSplitter(ll, sn)
 529     string_paren_wrap = StringParenWrapper(ll, sn)
 530
 531     transformers: List[Transformer]
 532     if (
 533         not line.contains_uncollapsable_type_comments()
 534         and not line.should_split_rhs
 535         and not line.magic_trailing_comma
 536         and (
 537             is_line_short_enough(line, mode=mode, line_str=line_str)
 538             or line.contains_unsplittable_type_ignore()
 539         )
 540         and not (line.inside_brackets and line.contains_standalone_comments())
 541     ):
 542         # Only apply basic string preprocessing, since lines shouldn't be split here.
 543         if Preview.string_processing in mode:
 544             transformers = [string_merge, string_paren_strip]
 545         else:
 546             transformers = []
 547     elif line.is_def:
 548         transformers = [left_hand_split]
 549     else:
 550
 551         def _rhs(
 552             self: object, line: Line, features: Collection[Feature], mode: Mode
 553         ) -> Iterator[Line]:
 554             """Wraps calls to `right_hand_split`.
 555
 556             The calls increasingly `omit` right-hand trailers (bracket pairs with
 557             content), meaning the trailers get glued together to split on another
 558             bracket pair instead.
 559             """
 560             for omit in generate_trailers_to_omit(line, mode.line_length):
 561                 lines = list(right_hand_split(line, mode, features, omit=omit))
 562                 # Note: this check is only able to figure out if the first line of the
 563                 # *current* transformation fits in the line length.  This is true only
 564                 # for simple cases.  All others require running more transforms via
 565                 # `transform_line()`.  This check doesn't know if those would succeed.
 566                 if is_line_short_enough(lines[0], mode=mode):
 567                     yield from lines
 568                     return
 569
 570             # All splits failed, best effort split with no omits.
 571             # This mostly happens to multiline strings that are by definition
 572             # reported as not fitting a single line, as well as lines that contain
 573             # trailing commas (those have to be exploded).
 574             yield from right_hand_split(line, mode, features=features)
 575
 576         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 577         # __name__ attribute which is needed in `run_transformer` further down.
 578         # Unfortunately a nested class breaks mypyc too. So a class must be created
 579         # via type ... https://github.com/mypyc/mypyc/issues/884
 580         rhs = type("rhs", (), {"__call__": _rhs})()
 581
 582         if Preview.string_processing in mode:
 583             if line.inside_brackets:
 584                 transformers = [
 585                     string_merge,
 586                     string_paren_strip,
 587                     string_split,
 588                     delimiter_split,
 589                     standalone_comment_split,
 590                     string_paren_wrap,
 591                     rhs,
 592                 ]
 593             else:
 594                 transformers = [
 595                     string_merge,
 596                     string_paren_strip,
 597                     string_split,
 598                     string_paren_wrap,
 599                     rhs,
 600                 ]
 601         else:
 602             if line.inside_brackets:
 603                 transformers = [delimiter_split, standalone_comment_split, rhs]
 604             else:
 605                 transformers = [rhs]
 606     # It's always safe to attempt hugging of power operations and pretty much every line
 607     # could match.
 608     transformers.append(hug_power_op)
 609
 610     for transform in transformers:
 611         # We are accumulating lines in `result` because we might want to abort
 612         # mission and return the original line in the end, or attempt a different
 613         # split altogether.
 614         try:
 615             result = run_transformer(line, transform, mode, features, line_str=line_str)
 616         except CannotTransform:
 617             continue
 618         else:
 619             yield from result
 620             break
 621
 622     else:
 623         yield line
 624
 625
 626 class _BracketSplitComponent(Enum):
 627     head = auto()
 628     body = auto()
 629     tail = auto()
 630
 631
 632 def left_hand_split(
 633     line: Line, _features: Collection[Feature], mode: Mode
 634 ) -> Iterator[Line]:
 635     """Split line into many lines, starting with the first matching bracket pair.
 636
 637     Note: this usually looks weird, only use this for function definitions.
 638     Prefer RHS otherwise.  This is why this function is not symmetrical with
 639     :func:`right_hand_split` which also handles optional parentheses.
 640     """
 641     tail_leaves: List[Leaf] = []
 642     body_leaves: List[Leaf] = []
 643     head_leaves: List[Leaf] = []
 644     current_leaves = head_leaves
 645     matching_bracket: Optional[Leaf] = None
 646     for leaf in line.leaves:
 647         if (
 648             current_leaves is body_leaves
 649             and leaf.type in CLOSING_BRACKETS
 650             and leaf.opening_bracket is matching_bracket
 651             and isinstance(matching_bracket, Leaf)
 652         ):
 653             ensure_visible(leaf)
 654             ensure_visible(matching_bracket)
 655             current_leaves = tail_leaves if body_leaves else head_leaves
 656         current_leaves.append(leaf)
 657         if current_leaves is head_leaves:
 658             if leaf.type in OPENING_BRACKETS:
 659                 matching_bracket = leaf
 660                 current_leaves = body_leaves
 661     if not matching_bracket:
 662         raise CannotSplit("No brackets found")
 663
 664     head = bracket_split_build_line(
 665         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 666     )
 667     body = bracket_split_build_line(
 668         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 669     )
 670     tail = bracket_split_build_line(
 671         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 672     )
 673     bracket_split_succeeded_or_raise(head, body, tail)
 674     for result in (head, body, tail):
 675         if result:
 676             yield result
 677
 678
 679 def right_hand_split(
 680     line: Line,
 681     mode: Mode,
 682     features: Collection[Feature] = (),
 683     omit: Collection[LeafID] = (),
 684 ) -> Iterator[Line]:
 685     """Split line into many lines, starting with the last matching bracket pair.
 686
 687     If the split was by optional parentheses, attempt splitting without them, too.
 688     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 689     this split.
 690
 691     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 692     """
 693     rhs_result = _first_right_hand_split(line, omit=omit)
 694     yield from _maybe_split_omitting_optional_parens(
 695         rhs_result, line, mode, features=features, omit=omit
 696     )
 697
 698
 699 def _first_right_hand_split(
 700     line: Line,
 701     omit: Collection[LeafID] = (),
 702 ) -> RHSResult:
 703     """Split the line into head, body, tail starting with the last bracket pair.
 704
 705     Note: this function should not have side effects. It's relied upon by
 706     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 707     splitting on the right side of an assignment statement.
 708     """
 709     tail_leaves: List[Leaf] = []
 710     body_leaves: List[Leaf] = []
 711     head_leaves: List[Leaf] = []
 712     current_leaves = tail_leaves
 713     opening_bracket: Optional[Leaf] = None
 714     closing_bracket: Optional[Leaf] = None
 715     for leaf in reversed(line.leaves):
 716         if current_leaves is body_leaves:
 717             if leaf is opening_bracket:
 718                 current_leaves = head_leaves if body_leaves else tail_leaves
 719         current_leaves.append(leaf)
 720         if current_leaves is tail_leaves:
 721             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 722                 opening_bracket = leaf.opening_bracket
 723                 closing_bracket = leaf
 724                 current_leaves = body_leaves
 725     if not (opening_bracket and closing_bracket and head_leaves):
 726         # If there is no opening or closing_bracket that means the split failed and
 727         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 728         # the matching `opening_bracket` wasn't available on `line` anymore.
 729         raise CannotSplit("No brackets found")
 730
 731     tail_leaves.reverse()
 732     body_leaves.reverse()
 733     head_leaves.reverse()
 734     head = bracket_split_build_line(
 735         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 736     )
 737     body = bracket_split_build_line(
 738         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 739     )
 740     tail = bracket_split_build_line(
 741         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 742     )
 743     bracket_split_succeeded_or_raise(head, body, tail)
 744     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 745
 746
 747 def _maybe_split_omitting_optional_parens(
 748     rhs: RHSResult,
 749     line: Line,
 750     mode: Mode,
 751     features: Collection[Feature] = (),
 752     omit: Collection[LeafID] = (),
 753 ) -> Iterator[Line]:
 754     if (
 755         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 756         # the opening bracket is an optional paren
 757         and rhs.opening_bracket.type == token.LPAR
 758         and not rhs.opening_bracket.value
 759         # the closing bracket is an optional paren
 760         and rhs.closing_bracket.type == token.RPAR
 761         and not rhs.closing_bracket.value
 762         # it's not an import (optional parens are the only thing we can split on
 763         # in this case; attempting a split without them is a waste of time)
 764         and not line.is_import
 765         # there are no standalone comments in the body
 766         and not rhs.body.contains_standalone_comments(0)
 767         # and we can actually remove the parens
 768         and can_omit_invisible_parens(rhs, mode.line_length)
 769     ):
 770         omit = {id(rhs.closing_bracket), *omit}
 771         try:
 772             # The RHSResult Omitting Optional Parens.
 773             rhs_oop = _first_right_hand_split(line, omit=omit)
 774             if not (
 775                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 776                 # the split is right after `=`
 777                 and len(rhs.head.leaves) >= 2
 778                 and rhs.head.leaves[-2].type == token.EQUAL
 779                 # the left side of assignment contains brackets
 780                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 781                 # the left side of assignment is short enough (the -1 is for the ending
 782                 # optional paren)
 783                 and is_line_short_enough(
 784                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 785                 )
 786                 # the left side of assignment won't explode further because of magic
 787                 # trailing comma
 788                 and rhs.head.magic_trailing_comma is None
 789                 # the split by omitting optional parens isn't preferred by some other
 790                 # reason
 791                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 792             ):
 793                 yield from _maybe_split_omitting_optional_parens(
 794                     rhs_oop, line, mode, features=features, omit=omit
 795                 )
 796                 return
 797
 798         except CannotSplit as e:
 799             if not (
 800                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 801             ):
 802                 raise CannotSplit(
 803                     "Splitting failed, body is still too long and can't be split."
 804                 ) from e
 805
 806             elif (
 807                 rhs.head.contains_multiline_strings()
 808                 or rhs.tail.contains_multiline_strings()
 809             ):
 810                 raise CannotSplit(
 811                     "The current optional pair of parentheses is bound to fail to"
 812                     " satisfy the splitting algorithm because the head or the tail"
 813                     " contains multiline strings which by definition never fit one"
 814                     " line."
 815                 ) from e
 816
 817     ensure_visible(rhs.opening_bracket)
 818     ensure_visible(rhs.closing_bracket)
 819     for result in (rhs.head, rhs.body, rhs.tail):
 820         if result:
 821             yield result
 822
 823
 824 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 825     """
 826     Returns whether we should prefer the result from a split omitting optional parens.
 827     """
 828     has_closing_bracket_after_assign = False
 829     for leaf in reversed(rhs_oop.head.leaves):
 830         if leaf.type == token.EQUAL:
 831             break
 832         if leaf.type in CLOSING_BRACKETS:
 833             has_closing_bracket_after_assign = True
 834             break
 835     return (
 836         # contains matching brackets after the `=` (done by checking there is a
 837         # closing bracket)
 838         has_closing_bracket_after_assign
 839         or (
 840             # the split is actually from inside the optional parens (done by checking
 841             # the first line still contains the `=`)
 842             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 843             # the first line is short enough
 844             and is_line_short_enough(rhs_oop.head, mode=mode)
 845         )
 846         # contains unsplittable type ignore
 847         or rhs_oop.head.contains_unsplittable_type_ignore()
 848         or rhs_oop.body.contains_unsplittable_type_ignore()
 849         or rhs_oop.tail.contains_unsplittable_type_ignore()
 850     )
 851
 852
 853 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 854     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 855
 856     Do nothing otherwise.
 857
 858     A left- or right-hand split is based on a pair of brackets. Content before
 859     (and including) the opening bracket is left on one line, content inside the
 860     brackets is put on a separate line, and finally content starting with and
 861     following the closing bracket is put on a separate line.
 862
 863     Those are called `head`, `body`, and `tail`, respectively. If the split
 864     produced the same line (all content in `head`) or ended up with an empty `body`
 865     and the `tail` is just the closing bracket, then it's considered failed.
 866     """
 867     tail_len = len(str(tail).strip())
 868     if not body:
 869         if tail_len == 0:
 870             raise CannotSplit("Splitting brackets produced the same line")
 871
 872         elif tail_len < 3:
 873             raise CannotSplit(
 874                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 875                 " not worth it"
 876             )
 877
 878
 879 def bracket_split_build_line(
 880     leaves: List[Leaf],
 881     original: Line,
 882     opening_bracket: Leaf,
 883     *,
 884     component: _BracketSplitComponent,
 885 ) -> Line:
 886     """Return a new line with given `leaves` and respective comments from `original`.
 887
 888     If it's the head component, brackets will be tracked so trailing commas are
 889     respected.
 890
 891     If it's the body component, the result line is one-indented inside brackets and as
 892     such has its first leaf's prefix normalized and a trailing comma added when
 893     expected.
 894     """
 895     result = Line(mode=original.mode, depth=original.depth)
 896     if component is _BracketSplitComponent.body:
 897         result.inside_brackets = True
 898         result.depth += 1
 899         if leaves:
 900             # Since body is a new indent level, remove spurious leading whitespace.
 901             normalize_prefix(leaves[0], inside_brackets=True)
 902             # Ensure a trailing comma for imports and standalone function arguments, but
 903             # be careful not to add one after any comments or within type annotations.
 904             no_commas = (
 905                 original.is_def
 906                 and opening_bracket.value == "("
 907                 and not any(leaf.type == token.COMMA for leaf in leaves)
 908                 # In particular, don't add one within a parenthesized return annotation.
 909                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 910                 # be defined directly in the parent node, the parent of the parent ...
 911                 # and so on depending on how complex the return annotation is.
 912                 # This isn't perfect and there's some false negatives but they are in
 913                 # contexts were a comma is actually fine.
 914                 and not any(
 915                     node.prev_sibling.type == RARROW
 916                     for node in (
 917                         leaves[0].parent,
 918                         getattr(leaves[0].parent, "parent", None),
 919                     )
 920                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 921                 )
 922                 # Except the false negatives above for PEP 604 unions where we
 923                 # can't add the comma.
 924                 and not (
 925                     leaves[0].parent
 926                     and leaves[0].parent.next_sibling
 927                     and leaves[0].parent.next_sibling.type == token.VBAR
 928                 )
 929             )
 930
 931             if original.is_import or no_commas:
 932                 for i in range(len(leaves) - 1, -1, -1):
 933                     if leaves[i].type == STANDALONE_COMMENT:
 934                         continue
 935
 936                     if leaves[i].type != token.COMMA:
 937                         new_comma = Leaf(token.COMMA, ",")
 938                         leaves.insert(i + 1, new_comma)
 939                     break
 940
 941     leaves_to_track: Set[LeafID] = set()
 942     if component is _BracketSplitComponent.head:
 943         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
 944     # Populate the line
 945     for leaf in leaves:
 946         result.append(
 947             leaf,
 948             preformatted=True,
 949             track_bracket=id(leaf) in leaves_to_track,
 950         )
 951         for comment_after in original.comments_after(leaf):
 952             result.append(comment_after, preformatted=True)
 953     if component is _BracketSplitComponent.body and should_split_line(
 954         result, opening_bracket
 955     ):
 956         result.should_split_rhs = True
 957     return result
 958
 959
 960 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 961     """Normalize prefix of the first leaf in every line returned by `split_func`.
 962
 963     This is a decorator over relevant split functions.
 964     """
 965
 966     @wraps(split_func)
 967     def split_wrapper(
 968         line: Line, features: Collection[Feature], mode: Mode
 969     ) -> Iterator[Line]:
 970         for split_line in split_func(line, features, mode):
 971             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 972             yield split_line
 973
 974     return split_wrapper
 975
 976
 977 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
 978     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
 979         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
 980             return leaf_idx
 981     return None
 982
 983
 984 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
 985     if (
 986         safe
 987         and delimiter_priority == COMMA_PRIORITY
 988         and line.leaves[-1].type != token.COMMA
 989         and line.leaves[-1].type != STANDALONE_COMMENT
 990     ):
 991         new_comma = Leaf(token.COMMA, ",")
 992         line.append(new_comma)
 993     return line
 994
 995
 996 @dont_increase_indentation
 997 def delimiter_split(
 998     line: Line, features: Collection[Feature], mode: Mode
 999 ) -> Iterator[Line]:
1000     """Split according to delimiters of the highest priority.
1001
1002     If the appropriate Features are given, the split will add trailing commas
1003     also in function signatures and calls that contain `*` and `**`.
1004     """
1005     try:
1006         last_leaf = line.leaves[-1]
1007     except IndexError:
1008         raise CannotSplit("Line empty") from None
1009
1010     bt = line.bracket_tracker
1011     try:
1012         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1013     except ValueError:
1014         raise CannotSplit("No delimiters found") from None
1015
1016     if delimiter_priority == DOT_PRIORITY:
1017         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
1018             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1019
1020     current_line = Line(
1021         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1022     )
1023     lowest_depth = sys.maxsize
1024     trailing_comma_safe = True
1025
1026     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1027         """Append `leaf` to current line or to new line if appending impossible."""
1028         nonlocal current_line
1029         try:
1030             current_line.append_safe(leaf, preformatted=True)
1031         except ValueError:
1032             yield current_line
1033
1034             current_line = Line(
1035                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1036             )
1037             current_line.append(leaf)
1038
1039     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1040     for leaf_idx, leaf in enumerate(line.leaves):
1041         yield from append_to_line(leaf)
1042
1043         for comment_after in line.comments_after(leaf):
1044             yield from append_to_line(comment_after)
1045
1046         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1047         if leaf.bracket_depth == lowest_depth:
1048             if is_vararg(leaf, within={syms.typedargslist}):
1049                 trailing_comma_safe = (
1050                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1051                 )
1052             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1053                 trailing_comma_safe = (
1054                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1055                 )
1056
1057         if (
1058             Preview.add_trailing_comma_consistently in mode
1059             and last_leaf.type == STANDALONE_COMMENT
1060             and leaf_idx == last_non_comment_leaf
1061         ):
1062             current_line = _safe_add_trailing_comma(
1063                 trailing_comma_safe, delimiter_priority, current_line
1064             )
1065
1066         leaf_priority = bt.delimiters.get(id(leaf))
1067         if leaf_priority == delimiter_priority:
1068             yield current_line
1069
1070             current_line = Line(
1071                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1072             )
1073     if current_line:
1074         current_line = _safe_add_trailing_comma(
1075             trailing_comma_safe, delimiter_priority, current_line
1076         )
1077         yield current_line
1078
1079
1080 @dont_increase_indentation
1081 def standalone_comment_split(
1082     line: Line, features: Collection[Feature], mode: Mode
1083 ) -> Iterator[Line]:
1084     """Split standalone comments from the rest of the line."""
1085     if not line.contains_standalone_comments(0):
1086         raise CannotSplit("Line does not have any standalone comments")
1087
1088     current_line = Line(
1089         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1090     )
1091
1092     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1093         """Append `leaf` to current line or to new line if appending impossible."""
1094         nonlocal current_line
1095         try:
1096             current_line.append_safe(leaf, preformatted=True)
1097         except ValueError:
1098             yield current_line
1099
1100             current_line = Line(
1101                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1102             )
1103             current_line.append(leaf)
1104
1105     for leaf in line.leaves:
1106         yield from append_to_line(leaf)
1107
1108         for comment_after in line.comments_after(leaf):
1109             yield from append_to_line(comment_after)
1110
1111     if current_line:
1112         yield current_line
1113
1114
1115 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1116     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1117     else.
1118
1119     Note: don't use backslashes for formatting or you'll lose your voting rights.
1120     """
1121     if not inside_brackets:
1122         spl = leaf.prefix.split("#")
1123         if "\\" not in spl[0]:
1124             nl_count = spl[-1].count("\n")
1125             if len(spl) > 1:
1126                 nl_count -= 1
1127             leaf.prefix = "\n" * nl_count
1128             return
1129
1130     leaf.prefix = ""
1131
1132
1133 def normalize_invisible_parens(
1134     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1135 ) -> None:
1136     """Make existing optional parentheses invisible or create new ones.
1137
1138     `parens_after` is a set of string leaf values immediately after which parens
1139     should be put.
1140
1141     Standardizes on visible parentheses for single-element tuples, and keeps
1142     existing visible parentheses for other tuples and generator expressions.
1143     """
1144     for pc in list_comments(node.prefix, is_endmarker=False):
1145         if pc.value in FMT_OFF:
1146             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1147             return
1148
1149     # The multiple context managers grammar has a different pattern, thus this is
1150     # separate from the for-loop below. This possibly wraps them in invisible parens,
1151     # and later will be removed in remove_with_parens when needed.
1152     if node.type == syms.with_stmt:
1153         _maybe_wrap_cms_in_parens(node, mode, features)
1154
1155     check_lpar = False
1156     for index, child in enumerate(list(node.children)):
1157         # Fixes a bug where invisible parens are not properly stripped from
1158         # assignment statements that contain type annotations.
1159         if isinstance(child, Node) and child.type == syms.annassign:
1160             normalize_invisible_parens(
1161                 child, parens_after=parens_after, mode=mode, features=features
1162             )
1163
1164         # Add parentheses around long tuple unpacking in assignments.
1165         if (
1166             index == 0
1167             and isinstance(child, Node)
1168             and child.type == syms.testlist_star_expr
1169         ):
1170             check_lpar = True
1171
1172         if check_lpar:
1173             if (
1174                 child.type == syms.atom
1175                 and node.type == syms.for_stmt
1176                 and isinstance(child.prev_sibling, Leaf)
1177                 and child.prev_sibling.type == token.NAME
1178                 and child.prev_sibling.value == "for"
1179             ):
1180                 if maybe_make_parens_invisible_in_atom(
1181                     child,
1182                     parent=node,
1183                     remove_brackets_around_comma=True,
1184                 ):
1185                     wrap_in_parentheses(node, child, visible=False)
1186             elif isinstance(child, Node) and node.type == syms.with_stmt:
1187                 remove_with_parens(child, node)
1188             elif child.type == syms.atom:
1189                 if maybe_make_parens_invisible_in_atom(
1190                     child,
1191                     parent=node,
1192                 ):
1193                     wrap_in_parentheses(node, child, visible=False)
1194             elif is_one_tuple(child):
1195                 wrap_in_parentheses(node, child, visible=True)
1196             elif node.type == syms.import_from:
1197                 _normalize_import_from(node, child, index)
1198                 break
1199             elif (
1200                 index == 1
1201                 and child.type == token.STAR
1202                 and node.type == syms.except_clause
1203             ):
1204                 # In except* (PEP 654), the star is actually part of
1205                 # of the keyword. So we need to skip the insertion of
1206                 # invisible parentheses to work more precisely.
1207                 continue
1208
1209             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1210                 wrap_in_parentheses(node, child, visible=False)
1211
1212         comma_check = child.type == token.COMMA
1213
1214         check_lpar = isinstance(child, Leaf) and (
1215             child.value in parens_after or comma_check
1216         )
1217
1218
1219 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1220     # "import from" nodes store parentheses directly as part of
1221     # the statement
1222     if is_lpar_token(child):
1223         assert is_rpar_token(parent.children[-1])
1224         # make parentheses invisible
1225         child.value = ""
1226         parent.children[-1].value = ""
1227     elif child.type != token.STAR:
1228         # insert invisible parentheses
1229         parent.insert_child(index, Leaf(token.LPAR, ""))
1230         parent.append_child(Leaf(token.RPAR, ""))
1231
1232
1233 def remove_await_parens(node: Node) -> None:
1234     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1235         if (
1236             node.children[1].type == syms.atom
1237             and node.children[1].children[0].type == token.LPAR
1238         ):
1239             if maybe_make_parens_invisible_in_atom(
1240                 node.children[1],
1241                 parent=node,
1242                 remove_brackets_around_comma=True,
1243             ):
1244                 wrap_in_parentheses(node, node.children[1], visible=False)
1245
1246             # Since await is an expression we shouldn't remove
1247             # brackets in cases where this would change
1248             # the AST due to operator precedence.
1249             # Therefore we only aim to remove brackets around
1250             # power nodes that aren't also await expressions themselves.
1251             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1252             # N.B. We've still removed any redundant nested brackets though :)
1253             opening_bracket = cast(Leaf, node.children[1].children[0])
1254             closing_bracket = cast(Leaf, node.children[1].children[-1])
1255             bracket_contents = node.children[1].children[1]
1256             if isinstance(bracket_contents, Node):
1257                 if bracket_contents.type != syms.power:
1258                     ensure_visible(opening_bracket)
1259                     ensure_visible(closing_bracket)
1260                 elif (
1261                     bracket_contents.type == syms.power
1262                     and bracket_contents.children[0].type == token.AWAIT
1263                 ):
1264                     ensure_visible(opening_bracket)
1265                     ensure_visible(closing_bracket)
1266                     # If we are in a nested await then recurse down.
1267                     remove_await_parens(bracket_contents)
1268
1269
1270 def _maybe_wrap_cms_in_parens(
1271     node: Node, mode: Mode, features: Collection[Feature]
1272 ) -> None:
1273     """When enabled and safe, wrap the multiple context managers in invisible parens.
1274
1275     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1276     """
1277     if (
1278         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1279         or Preview.wrap_multiple_context_managers_in_parens not in mode
1280         or len(node.children) <= 2
1281         # If it's an atom, it's already wrapped in parens.
1282         or node.children[1].type == syms.atom
1283     ):
1284         return
1285     colon_index: Optional[int] = None
1286     for i in range(2, len(node.children)):
1287         if node.children[i].type == token.COLON:
1288             colon_index = i
1289             break
1290     if colon_index is not None:
1291         lpar = Leaf(token.LPAR, "")
1292         rpar = Leaf(token.RPAR, "")
1293         context_managers = node.children[1:colon_index]
1294         for child in context_managers:
1295             child.remove()
1296         # After wrapping, the with_stmt will look like this:
1297         #   with_stmt
1298         #     NAME 'with'
1299         #     atom
1300         #       LPAR ''
1301         #       testlist_gexp
1302         #         ... <-- context_managers
1303         #       /testlist_gexp
1304         #       RPAR ''
1305         #     /atom
1306         #     COLON ':'
1307         new_child = Node(
1308             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1309         )
1310         node.insert_child(1, new_child)
1311
1312
1313 def remove_with_parens(node: Node, parent: Node) -> None:
1314     """Recursively hide optional parens in `with` statements."""
1315     # Removing all unnecessary parentheses in with statements in one pass is a tad
1316     # complex as different variations of bracketed statements result in pretty
1317     # different parse trees:
1318     #
1319     # with (open("file")) as f:                       # this is an asexpr_test
1320     #     ...
1321     #
1322     # with (open("file") as f):                       # this is an atom containing an
1323     #     ...                                         # asexpr_test
1324     #
1325     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1326     #     ...                                         # asexpr_test
1327     #
1328     # with (open("file") as f, open("file") as f):    # an atom containing a
1329     #     ...                                         # testlist_gexp which then
1330     #                                                 # contains multiple asexpr_test(s)
1331     if node.type == syms.atom:
1332         if maybe_make_parens_invisible_in_atom(
1333             node,
1334             parent=parent,
1335             remove_brackets_around_comma=True,
1336         ):
1337             wrap_in_parentheses(parent, node, visible=False)
1338         if isinstance(node.children[1], Node):
1339             remove_with_parens(node.children[1], node)
1340     elif node.type == syms.testlist_gexp:
1341         for child in node.children:
1342             if isinstance(child, Node):
1343                 remove_with_parens(child, node)
1344     elif node.type == syms.asexpr_test and not any(
1345         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1346     ):
1347         if maybe_make_parens_invisible_in_atom(
1348             node.children[0],
1349             parent=node,
1350             remove_brackets_around_comma=True,
1351         ):
1352             wrap_in_parentheses(node, node.children[0], visible=False)
1353
1354
1355 def maybe_make_parens_invisible_in_atom(
1356     node: LN,
1357     parent: LN,
1358     remove_brackets_around_comma: bool = False,
1359 ) -> bool:
1360     """If it's safe, make the parens in the atom `node` invisible, recursively.
1361     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1362     as they are redundant.
1363
1364     Returns whether the node should itself be wrapped in invisible parentheses.
1365     """
1366     if (
1367         node.type != syms.atom
1368         or is_empty_tuple(node)
1369         or is_one_tuple(node)
1370         or (is_yield(node) and parent.type != syms.expr_stmt)
1371         or (
1372             # This condition tries to prevent removing non-optional brackets
1373             # around a tuple, however, can be a bit overzealous so we provide
1374             # and option to skip this check for `for` and `with` statements.
1375             not remove_brackets_around_comma
1376             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1377         )
1378         or is_tuple_containing_walrus(node)
1379     ):
1380         return False
1381
1382     if is_walrus_assignment(node):
1383         if parent.type in [
1384             syms.annassign,
1385             syms.expr_stmt,
1386             syms.assert_stmt,
1387             syms.return_stmt,
1388             syms.except_clause,
1389             syms.funcdef,
1390             syms.with_stmt,
1391             # these ones aren't useful to end users, but they do please fuzzers
1392             syms.for_stmt,
1393             syms.del_stmt,
1394             syms.for_stmt,
1395         ]:
1396             return False
1397
1398     first = node.children[0]
1399     last = node.children[-1]
1400     if is_lpar_token(first) and is_rpar_token(last):
1401         middle = node.children[1]
1402         # make parentheses invisible
1403         if (
1404             # If the prefix of `middle` includes a type comment with
1405             # ignore annotation, then we do not remove the parentheses
1406             not is_type_ignore_comment_string(middle.prefix.strip())
1407         ):
1408             first.value = ""
1409             last.value = ""
1410         maybe_make_parens_invisible_in_atom(
1411             middle,
1412             parent=parent,
1413             remove_brackets_around_comma=remove_brackets_around_comma,
1414         )
1415
1416         if is_atom_with_invisible_parens(middle):
1417             # Strip the invisible parens from `middle` by replacing
1418             # it with the child in-between the invisible parens
1419             middle.replace(middle.children[1])
1420
1421         return False
1422
1423     return True
1424
1425
1426 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1427     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1428
1429     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1430         return False
1431
1432     # We're essentially checking if the body is delimited by commas and there's more
1433     # than one of them (we're excluding the trailing comma and if the delimiter priority
1434     # is still commas, that means there's more).
1435     exclude = set()
1436     trailing_comma = False
1437     try:
1438         last_leaf = line.leaves[-1]
1439         if last_leaf.type == token.COMMA:
1440             trailing_comma = True
1441             exclude.add(id(last_leaf))
1442         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1443     except (IndexError, ValueError):
1444         return False
1445
1446     return max_priority == COMMA_PRIORITY and (
1447         (line.mode.magic_trailing_comma and trailing_comma)
1448         # always explode imports
1449         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1450     )
1451
1452
1453 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1454     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1455
1456     Brackets can be omitted if the entire trailer up to and including
1457     a preceding closing bracket fits in one line.
1458
1459     Yielded sets are cumulative (contain results of previous yields, too).  First
1460     set is empty, unless the line should explode, in which case bracket pairs until
1461     the one that needs to explode are omitted.
1462     """
1463
1464     omit: Set[LeafID] = set()
1465     if not line.magic_trailing_comma:
1466         yield omit
1467
1468     length = 4 * line.depth
1469     opening_bracket: Optional[Leaf] = None
1470     closing_bracket: Optional[Leaf] = None
1471     inner_brackets: Set[LeafID] = set()
1472     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1473         length += leaf_length
1474         if length > line_length:
1475             break
1476
1477         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1478         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1479             break
1480
1481         if opening_bracket:
1482             if leaf is opening_bracket:
1483                 opening_bracket = None
1484             elif leaf.type in CLOSING_BRACKETS:
1485                 prev = line.leaves[index - 1] if index > 0 else None
1486                 if (
1487                     prev
1488                     and prev.type == token.COMMA
1489                     and leaf.opening_bracket is not None
1490                     and not is_one_sequence_between(
1491                         leaf.opening_bracket, leaf, line.leaves
1492                     )
1493                 ):
1494                     # Never omit bracket pairs with trailing commas.
1495                     # We need to explode on those.
1496                     break
1497
1498                 inner_brackets.add(id(leaf))
1499         elif leaf.type in CLOSING_BRACKETS:
1500             prev = line.leaves[index - 1] if index > 0 else None
1501             if prev and prev.type in OPENING_BRACKETS:
1502                 # Empty brackets would fail a split so treat them as "inner"
1503                 # brackets (e.g. only add them to the `omit` set if another
1504                 # pair of brackets was good enough.
1505                 inner_brackets.add(id(leaf))
1506                 continue
1507
1508             if closing_bracket:
1509                 omit.add(id(closing_bracket))
1510                 omit.update(inner_brackets)
1511                 inner_brackets.clear()
1512                 yield omit
1513
1514             if (
1515                 prev
1516                 and prev.type == token.COMMA
1517                 and leaf.opening_bracket is not None
1518                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1519             ):
1520                 # Never omit bracket pairs with trailing commas.
1521                 # We need to explode on those.
1522                 break
1523
1524             if leaf.value:
1525                 opening_bracket = leaf.opening_bracket
1526                 closing_bracket = leaf
1527
1528
1529 def run_transformer(
1530     line: Line,
1531     transform: Transformer,
1532     mode: Mode,
1533     features: Collection[Feature],
1534     *,
1535     line_str: str = "",
1536 ) -> List[Line]:
1537     if not line_str:
1538         line_str = line_to_string(line)
1539     result: List[Line] = []
1540     for transformed_line in transform(line, features, mode):
1541         if str(transformed_line).strip("\n") == line_str:
1542             raise CannotTransform("Line transformer returned an unchanged result")
1543
1544         result.extend(transform_line(transformed_line, mode=mode, features=features))
1545
1546     features_set = set(features)
1547     if (
1548         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1549         or transform.__class__.__name__ != "rhs"
1550         or not line.bracket_tracker.invisible
1551         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1552         or line.contains_multiline_strings()
1553         or result[0].contains_uncollapsable_type_comments()
1554         or result[0].contains_unsplittable_type_ignore()
1555         or is_line_short_enough(result[0], mode=mode)
1556         # If any leaves have no parents (which _can_ occur since
1557         # `transform(line)` potentially destroys the line's underlying node
1558         # structure), then we can't proceed. Doing so would cause the below
1559         # call to `append_leaves()` to fail.
1560         or any(leaf.parent is None for leaf in line.leaves)
1561     ):
1562         return result
1563
1564     line_copy = line.clone()
1565     append_leaves(line_copy, line, line.leaves)
1566     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1567     second_opinion = run_transformer(
1568         line_copy, transform, mode, features_fop, line_str=line_str
1569     )
1570     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1571         result = second_opinion
1572     return result