src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import replace
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     RHSResult,
  20     append_leaves,
  21     can_be_split,
  22     can_omit_invisible_parens,
  23     is_line_short_enough,
  24     line_to_string,
  25 )
  26 from black.mode import Feature, Mode, Preview
  27 from black.nodes import (
  28     ASSIGNMENTS,
  29     BRACKETS,
  30     CLOSING_BRACKETS,
  31     OPENING_BRACKETS,
  32     RARROW,
  33     STANDALONE_COMMENT,
  34     STATEMENT,
  35     WHITESPACE,
  36     Visitor,
  37     ensure_visible,
  38     is_arith_like,
  39     is_atom_with_invisible_parens,
  40     is_docstring,
  41     is_empty_tuple,
  42     is_lpar_token,
  43     is_multiline_string,
  44     is_name_token,
  45     is_one_sequence_between,
  46     is_one_tuple,
  47     is_rpar_token,
  48     is_stub_body,
  49     is_stub_suite,
  50     is_tuple_containing_walrus,
  51     is_vararg,
  52     is_walrus_assignment,
  53     is_yield,
  54     syms,
  55     wrap_in_parentheses,
  56 )
  57 from black.numerics import normalize_numeric_literal
  58 from black.strings import (
  59     fix_docstring,
  60     get_string_prefix,
  61     normalize_string_prefix,
  62     normalize_string_quotes,
  63     normalize_unicode_escape_sequences,
  64 )
  65 from black.trans import (
  66     CannotTransform,
  67     StringMerger,
  68     StringParenStripper,
  69     StringParenWrapper,
  70     StringSplitter,
  71     Transformer,
  72     hug_power_op,
  73 )
  74 from blib2to3.pgen2 import token
  75 from blib2to3.pytree import Leaf, Node
  76
  77 # types
  78 LeafID = int
  79 LN = Union[Leaf, Node]
  80
  81
  82 class CannotSplit(CannotTransform):
  83     """A readable split that fits the allotted line length is impossible."""
  84
  85
  86 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  87 # See also https://github.com/mypyc/mypyc/issues/827.
  88 class LineGenerator(Visitor[Line]):
  89     """Generates reformatted Line objects.  Empty lines are not emitted.
  90
  91     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  92     in ways that will no longer stringify to valid Python code on the tree.
  93     """
  94
  95     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  96         self.mode = mode
  97         self.features = features
  98         self.current_line: Line
  99         self.__post_init__()
 100
 101     def line(self, indent: int = 0) -> Iterator[Line]:
 102         """Generate a line.
 103
 104         If the line is empty, only emit if it makes sense.
 105         If the line is too long, split it first and then generate.
 106
 107         If any lines were generated, set up a new current_line.
 108         """
 109         if not self.current_line:
 110             self.current_line.depth += indent
 111             return  # Line is empty, don't emit. Creating a new one unnecessary.
 112
 113         complete_line = self.current_line
 114         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 115         yield complete_line
 116
 117     def visit_default(self, node: LN) -> Iterator[Line]:
 118         """Default `visit_*()` implementation. Recurses to children of `node`."""
 119         if isinstance(node, Leaf):
 120             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 121             for comment in generate_comments(node):
 122                 if any_open_brackets:
 123                     # any comment within brackets is subject to splitting
 124                     self.current_line.append(comment)
 125                 elif comment.type == token.COMMENT:
 126                     # regular trailing comment
 127                     self.current_line.append(comment)
 128                     yield from self.line()
 129
 130                 else:
 131                     # regular standalone comment
 132                     yield from self.line()
 133
 134                     self.current_line.append(comment)
 135                     yield from self.line()
 136
 137             normalize_prefix(node, inside_brackets=any_open_brackets)
 138             if self.mode.string_normalization and node.type == token.STRING:
 139                 node.value = normalize_string_prefix(node.value)
 140                 node.value = normalize_string_quotes(node.value)
 141             if node.type == token.NUMBER:
 142                 normalize_numeric_literal(node)
 143             if node.type not in WHITESPACE:
 144                 self.current_line.append(node)
 145         yield from super().visit_default(node)
 146
 147     def visit_test(self, node: Node) -> Iterator[Line]:
 148         """Visit an `x if y else z` test"""
 149
 150         if Preview.parenthesize_conditional_expressions in self.mode:
 151             already_parenthesized = (
 152                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 153             )
 154
 155             if not already_parenthesized:
 156                 lpar = Leaf(token.LPAR, "")
 157                 rpar = Leaf(token.RPAR, "")
 158                 node.insert_child(0, lpar)
 159                 node.append_child(rpar)
 160
 161         yield from self.visit_default(node)
 162
 163     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 164         """Increase indentation level, maybe yield a line."""
 165         # In blib2to3 INDENT never holds comments.
 166         yield from self.line(+1)
 167         yield from self.visit_default(node)
 168
 169     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 170         """Decrease indentation level, maybe yield a line."""
 171         # The current line might still wait for trailing comments.  At DEDENT time
 172         # there won't be any (they would be prefixes on the preceding NEWLINE).
 173         # Emit the line then.
 174         yield from self.line()
 175
 176         # While DEDENT has no value, its prefix may contain standalone comments
 177         # that belong to the current indentation level.  Get 'em.
 178         yield from self.visit_default(node)
 179
 180         # Finally, emit the dedent.
 181         yield from self.line(-1)
 182
 183     def visit_stmt(
 184         self, node: Node, keywords: Set[str], parens: Set[str]
 185     ) -> Iterator[Line]:
 186         """Visit a statement.
 187
 188         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 189         `def`, `with`, `class`, `assert`, and assignments.
 190
 191         The relevant Python language `keywords` for a given statement will be
 192         NAME leaves within it. This methods puts those on a separate line.
 193
 194         `parens` holds a set of string leaf values immediately after which
 195         invisible parens should be put.
 196         """
 197         normalize_invisible_parens(
 198             node, parens_after=parens, mode=self.mode, features=self.features
 199         )
 200         for child in node.children:
 201             if is_name_token(child) and child.value in keywords:
 202                 yield from self.line()
 203
 204             yield from self.visit(child)
 205
 206     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 207         if Preview.wrap_long_dict_values_in_parens in self.mode:
 208             for i, child in enumerate(node.children):
 209                 if i == 0:
 210                     continue
 211                 if node.children[i - 1].type == token.COLON:
 212                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 213                         if maybe_make_parens_invisible_in_atom(
 214                             child,
 215                             parent=node,
 216                             remove_brackets_around_comma=False,
 217                         ):
 218                             wrap_in_parentheses(node, child, visible=False)
 219                     else:
 220                         wrap_in_parentheses(node, child, visible=False)
 221         yield from self.visit_default(node)
 222
 223     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 224         """Visit function definition."""
 225         yield from self.line()
 226
 227         # Remove redundant brackets around return type annotation.
 228         is_return_annotation = False
 229         for child in node.children:
 230             if child.type == token.RARROW:
 231                 is_return_annotation = True
 232             elif is_return_annotation:
 233                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 234                     if maybe_make_parens_invisible_in_atom(
 235                         child,
 236                         parent=node,
 237                         remove_brackets_around_comma=False,
 238                     ):
 239                         wrap_in_parentheses(node, child, visible=False)
 240                 else:
 241                     wrap_in_parentheses(node, child, visible=False)
 242                 is_return_annotation = False
 243
 244         for child in node.children:
 245             yield from self.visit(child)
 246
 247     def visit_match_case(self, node: Node) -> Iterator[Line]:
 248         """Visit either a match or case statement."""
 249         normalize_invisible_parens(
 250             node, parens_after=set(), mode=self.mode, features=self.features
 251         )
 252
 253         yield from self.line()
 254         for child in node.children:
 255             yield from self.visit(child)
 256
 257     def visit_suite(self, node: Node) -> Iterator[Line]:
 258         """Visit a suite."""
 259         if self.mode.is_pyi and is_stub_suite(node):
 260             yield from self.visit(node.children[2])
 261         else:
 262             yield from self.visit_default(node)
 263
 264     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 265         """Visit a statement without nested statements."""
 266         prev_type: Optional[int] = None
 267         for child in node.children:
 268             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 269                 wrap_in_parentheses(node, child, visible=False)
 270             prev_type = child.type
 271
 272         is_suite_like = node.parent and node.parent.type in STATEMENT
 273         if is_suite_like:
 274             if self.mode.is_pyi and is_stub_body(node):
 275                 yield from self.visit_default(node)
 276             else:
 277                 yield from self.line(+1)
 278                 yield from self.visit_default(node)
 279                 yield from self.line(-1)
 280
 281         else:
 282             if (
 283                 not self.mode.is_pyi
 284                 or not node.parent
 285                 or not is_stub_suite(node.parent)
 286             ):
 287                 yield from self.line()
 288             yield from self.visit_default(node)
 289
 290     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 291         """Visit `async def`, `async for`, `async with`."""
 292         yield from self.line()
 293
 294         children = iter(node.children)
 295         for child in children:
 296             yield from self.visit(child)
 297
 298             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 299                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 300                 # line.
 301                 break
 302
 303         internal_stmt = next(children)
 304         for child in internal_stmt.children:
 305             yield from self.visit(child)
 306
 307     def visit_decorators(self, node: Node) -> Iterator[Line]:
 308         """Visit decorators."""
 309         for child in node.children:
 310             yield from self.line()
 311             yield from self.visit(child)
 312
 313     def visit_power(self, node: Node) -> Iterator[Line]:
 314         for idx, leaf in enumerate(node.children[:-1]):
 315             next_leaf = node.children[idx + 1]
 316
 317             if not isinstance(leaf, Leaf):
 318                 continue
 319
 320             value = leaf.value.lower()
 321             if (
 322                 leaf.type == token.NUMBER
 323                 and next_leaf.type == syms.trailer
 324                 # Ensure that we are in an attribute trailer
 325                 and next_leaf.children[0].type == token.DOT
 326                 # It shouldn't wrap hexadecimal, binary and octal literals
 327                 and not value.startswith(("0x", "0b", "0o"))
 328                 # It shouldn't wrap complex literals
 329                 and "j" not in value
 330             ):
 331                 wrap_in_parentheses(node, leaf)
 332
 333         remove_await_parens(node)
 334
 335         yield from self.visit_default(node)
 336
 337     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 338         """Remove a semicolon and put the other statement on a separate line."""
 339         yield from self.line()
 340
 341     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 342         """End of file. Process outstanding comments and end with a newline."""
 343         yield from self.visit_default(leaf)
 344         yield from self.line()
 345
 346     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 347         if not self.current_line.bracket_tracker.any_open_brackets():
 348             yield from self.line()
 349         yield from self.visit_default(leaf)
 350
 351     def visit_factor(self, node: Node) -> Iterator[Line]:
 352         """Force parentheses between a unary op and a binary power:
 353
 354         -2 ** 8 -> -(2 ** 8)
 355         """
 356         _operator, operand = node.children
 357         if (
 358             operand.type == syms.power
 359             and len(operand.children) == 3
 360             and operand.children[1].type == token.DOUBLESTAR
 361         ):
 362             lpar = Leaf(token.LPAR, "(")
 363             rpar = Leaf(token.RPAR, ")")
 364             index = operand.remove() or 0
 365             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 366         yield from self.visit_default(node)
 367
 368     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 369         if Preview.hex_codes_in_unicode_sequences in self.mode:
 370             normalize_unicode_escape_sequences(leaf)
 371
 372         if is_docstring(leaf) and "\\\n" not in leaf.value:
 373             # We're ignoring docstrings with backslash newline escapes because changing
 374             # indentation of those changes the AST representation of the code.
 375             if self.mode.string_normalization:
 376                 docstring = normalize_string_prefix(leaf.value)
 377                 # visit_default() does handle string normalization for us, but
 378                 # since this method acts differently depending on quote style (ex.
 379                 # see padding logic below), there's a possibility for unstable
 380                 # formatting as visit_default() is called *after*. To avoid a
 381                 # situation where this function formats a docstring differently on
 382                 # the second pass, normalize it early.
 383                 docstring = normalize_string_quotes(docstring)
 384             else:
 385                 docstring = leaf.value
 386             prefix = get_string_prefix(docstring)
 387             docstring = docstring[len(prefix) :]  # Remove the prefix
 388             quote_char = docstring[0]
 389             # A natural way to remove the outer quotes is to do:
 390             #   docstring = docstring.strip(quote_char)
 391             # but that breaks on """""x""" (which is '""x').
 392             # So we actually need to remove the first character and the next two
 393             # characters but only if they are the same as the first.
 394             quote_len = 1 if docstring[1] != quote_char else 3
 395             docstring = docstring[quote_len:-quote_len]
 396             docstring_started_empty = not docstring
 397             indent = " " * 4 * self.current_line.depth
 398
 399             if is_multiline_string(leaf):
 400                 docstring = fix_docstring(docstring, indent)
 401             else:
 402                 docstring = docstring.strip()
 403
 404             has_trailing_backslash = False
 405             if docstring:
 406                 # Add some padding if the docstring starts / ends with a quote mark.
 407                 if docstring[0] == quote_char:
 408                     docstring = " " + docstring
 409                 if docstring[-1] == quote_char:
 410                     docstring += " "
 411                 if docstring[-1] == "\\":
 412                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 413                     if backslash_count % 2:
 414                         # Odd number of tailing backslashes, add some padding to
 415                         # avoid escaping the closing string quote.
 416                         docstring += " "
 417                         has_trailing_backslash = True
 418             elif not docstring_started_empty:
 419                 docstring = " "
 420
 421             # We could enforce triple quotes at this point.
 422             quote = quote_char * quote_len
 423
 424             # It's invalid to put closing single-character quotes on a new line.
 425             if self.mode and quote_len == 3:
 426                 # We need to find the length of the last line of the docstring
 427                 # to find if we can add the closing quotes to the line without
 428                 # exceeding the maximum line length.
 429                 # If docstring is one line, we don't put the closing quotes on a
 430                 # separate line because it looks ugly (#3320).
 431                 lines = docstring.splitlines()
 432                 last_line_length = len(lines[-1]) if docstring else 0
 433
 434                 # If adding closing quotes would cause the last line to exceed
 435                 # the maximum line length then put a line break before the
 436                 # closing quotes
 437                 if (
 438                     len(lines) > 1
 439                     and last_line_length + quote_len > self.mode.line_length
 440                     and len(indent) + quote_len <= self.mode.line_length
 441                     and not has_trailing_backslash
 442                 ):
 443                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 444                 else:
 445                     leaf.value = prefix + quote + docstring + quote
 446             else:
 447                 leaf.value = prefix + quote + docstring + quote
 448
 449         yield from self.visit_default(leaf)
 450
 451     def __post_init__(self) -> None:
 452         """You are in a twisty little maze of passages."""
 453         self.current_line = Line(mode=self.mode)
 454
 455         v = self.visit_stmt
 456         Ø: Set[str] = set()
 457         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 458         self.visit_if_stmt = partial(
 459             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 460         )
 461         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 462         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 463         self.visit_try_stmt = partial(
 464             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 465         )
 466         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 467         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 468         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 469         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 470         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 471         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 472         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 473         self.visit_async_funcdef = self.visit_async_stmt
 474         self.visit_decorated = self.visit_decorators
 475
 476         # PEP 634
 477         self.visit_match_stmt = self.visit_match_case
 478         self.visit_case_block = self.visit_match_case
 479
 480
 481 def transform_line(
 482     line: Line, mode: Mode, features: Collection[Feature] = ()
 483 ) -> Iterator[Line]:
 484     """Transform a `line`, potentially splitting it into many lines.
 485
 486     They should fit in the allotted `line_length` but might not be able to.
 487
 488     `features` are syntactical features that may be used in the output.
 489     """
 490     if line.is_comment:
 491         yield line
 492         return
 493
 494     line_str = line_to_string(line)
 495
 496     ll = mode.line_length
 497     sn = mode.string_normalization
 498     string_merge = StringMerger(ll, sn)
 499     string_paren_strip = StringParenStripper(ll, sn)
 500     string_split = StringSplitter(ll, sn)
 501     string_paren_wrap = StringParenWrapper(ll, sn)
 502
 503     transformers: List[Transformer]
 504     if (
 505         not line.contains_uncollapsable_type_comments()
 506         and not line.should_split_rhs
 507         and not line.magic_trailing_comma
 508         and (
 509             is_line_short_enough(line, mode=mode, line_str=line_str)
 510             or line.contains_unsplittable_type_ignore()
 511         )
 512         and not (line.inside_brackets and line.contains_standalone_comments())
 513     ):
 514         # Only apply basic string preprocessing, since lines shouldn't be split here.
 515         if Preview.string_processing in mode:
 516             transformers = [string_merge, string_paren_strip]
 517         else:
 518             transformers = []
 519     elif line.is_def:
 520         transformers = [left_hand_split]
 521     else:
 522
 523         def _rhs(
 524             self: object, line: Line, features: Collection[Feature], mode: Mode
 525         ) -> Iterator[Line]:
 526             """Wraps calls to `right_hand_split`.
 527
 528             The calls increasingly `omit` right-hand trailers (bracket pairs with
 529             content), meaning the trailers get glued together to split on another
 530             bracket pair instead.
 531             """
 532             for omit in generate_trailers_to_omit(line, mode.line_length):
 533                 lines = list(right_hand_split(line, mode, features, omit=omit))
 534                 # Note: this check is only able to figure out if the first line of the
 535                 # *current* transformation fits in the line length.  This is true only
 536                 # for simple cases.  All others require running more transforms via
 537                 # `transform_line()`.  This check doesn't know if those would succeed.
 538                 if is_line_short_enough(lines[0], mode=mode):
 539                     yield from lines
 540                     return
 541
 542             # All splits failed, best effort split with no omits.
 543             # This mostly happens to multiline strings that are by definition
 544             # reported as not fitting a single line, as well as lines that contain
 545             # trailing commas (those have to be exploded).
 546             yield from right_hand_split(line, mode, features=features)
 547
 548         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 549         # __name__ attribute which is needed in `run_transformer` further down.
 550         # Unfortunately a nested class breaks mypyc too. So a class must be created
 551         # via type ... https://github.com/mypyc/mypyc/issues/884
 552         rhs = type("rhs", (), {"__call__": _rhs})()
 553
 554         if Preview.string_processing in mode:
 555             if line.inside_brackets:
 556                 transformers = [
 557                     string_merge,
 558                     string_paren_strip,
 559                     string_split,
 560                     delimiter_split,
 561                     standalone_comment_split,
 562                     string_paren_wrap,
 563                     rhs,
 564                 ]
 565             else:
 566                 transformers = [
 567                     string_merge,
 568                     string_paren_strip,
 569                     string_split,
 570                     string_paren_wrap,
 571                     rhs,
 572                 ]
 573         else:
 574             if line.inside_brackets:
 575                 transformers = [delimiter_split, standalone_comment_split, rhs]
 576             else:
 577                 transformers = [rhs]
 578     # It's always safe to attempt hugging of power operations and pretty much every line
 579     # could match.
 580     transformers.append(hug_power_op)
 581
 582     for transform in transformers:
 583         # We are accumulating lines in `result` because we might want to abort
 584         # mission and return the original line in the end, or attempt a different
 585         # split altogether.
 586         try:
 587             result = run_transformer(line, transform, mode, features, line_str=line_str)
 588         except CannotTransform:
 589             continue
 590         else:
 591             yield from result
 592             break
 593
 594     else:
 595         yield line
 596
 597
 598 class _BracketSplitComponent(Enum):
 599     head = auto()
 600     body = auto()
 601     tail = auto()
 602
 603
 604 def left_hand_split(
 605     line: Line, _features: Collection[Feature], mode: Mode
 606 ) -> Iterator[Line]:
 607     """Split line into many lines, starting with the first matching bracket pair.
 608
 609     Note: this usually looks weird, only use this for function definitions.
 610     Prefer RHS otherwise.  This is why this function is not symmetrical with
 611     :func:`right_hand_split` which also handles optional parentheses.
 612     """
 613     tail_leaves: List[Leaf] = []
 614     body_leaves: List[Leaf] = []
 615     head_leaves: List[Leaf] = []
 616     current_leaves = head_leaves
 617     matching_bracket: Optional[Leaf] = None
 618     for leaf in line.leaves:
 619         if (
 620             current_leaves is body_leaves
 621             and leaf.type in CLOSING_BRACKETS
 622             and leaf.opening_bracket is matching_bracket
 623             and isinstance(matching_bracket, Leaf)
 624         ):
 625             ensure_visible(leaf)
 626             ensure_visible(matching_bracket)
 627             current_leaves = tail_leaves if body_leaves else head_leaves
 628         current_leaves.append(leaf)
 629         if current_leaves is head_leaves:
 630             if leaf.type in OPENING_BRACKETS:
 631                 matching_bracket = leaf
 632                 current_leaves = body_leaves
 633     if not matching_bracket:
 634         raise CannotSplit("No brackets found")
 635
 636     head = bracket_split_build_line(
 637         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 638     )
 639     body = bracket_split_build_line(
 640         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 641     )
 642     tail = bracket_split_build_line(
 643         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 644     )
 645     bracket_split_succeeded_or_raise(head, body, tail)
 646     for result in (head, body, tail):
 647         if result:
 648             yield result
 649
 650
 651 def right_hand_split(
 652     line: Line,
 653     mode: Mode,
 654     features: Collection[Feature] = (),
 655     omit: Collection[LeafID] = (),
 656 ) -> Iterator[Line]:
 657     """Split line into many lines, starting with the last matching bracket pair.
 658
 659     If the split was by optional parentheses, attempt splitting without them, too.
 660     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 661     this split.
 662
 663     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 664     """
 665     rhs_result = _first_right_hand_split(line, omit=omit)
 666     yield from _maybe_split_omitting_optional_parens(
 667         rhs_result, line, mode, features=features, omit=omit
 668     )
 669
 670
 671 def _first_right_hand_split(
 672     line: Line,
 673     omit: Collection[LeafID] = (),
 674 ) -> RHSResult:
 675     """Split the line into head, body, tail starting with the last bracket pair.
 676
 677     Note: this function should not have side effects. It's relied upon by
 678     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 679     splitting on the right side of an assignment statement.
 680     """
 681     tail_leaves: List[Leaf] = []
 682     body_leaves: List[Leaf] = []
 683     head_leaves: List[Leaf] = []
 684     current_leaves = tail_leaves
 685     opening_bracket: Optional[Leaf] = None
 686     closing_bracket: Optional[Leaf] = None
 687     for leaf in reversed(line.leaves):
 688         if current_leaves is body_leaves:
 689             if leaf is opening_bracket:
 690                 current_leaves = head_leaves if body_leaves else tail_leaves
 691         current_leaves.append(leaf)
 692         if current_leaves is tail_leaves:
 693             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 694                 opening_bracket = leaf.opening_bracket
 695                 closing_bracket = leaf
 696                 current_leaves = body_leaves
 697     if not (opening_bracket and closing_bracket and head_leaves):
 698         # If there is no opening or closing_bracket that means the split failed and
 699         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 700         # the matching `opening_bracket` wasn't available on `line` anymore.
 701         raise CannotSplit("No brackets found")
 702
 703     tail_leaves.reverse()
 704     body_leaves.reverse()
 705     head_leaves.reverse()
 706     head = bracket_split_build_line(
 707         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 708     )
 709     body = bracket_split_build_line(
 710         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 711     )
 712     tail = bracket_split_build_line(
 713         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 714     )
 715     bracket_split_succeeded_or_raise(head, body, tail)
 716     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 717
 718
 719 def _maybe_split_omitting_optional_parens(
 720     rhs: RHSResult,
 721     line: Line,
 722     mode: Mode,
 723     features: Collection[Feature] = (),
 724     omit: Collection[LeafID] = (),
 725 ) -> Iterator[Line]:
 726     if (
 727         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 728         # the opening bracket is an optional paren
 729         and rhs.opening_bracket.type == token.LPAR
 730         and not rhs.opening_bracket.value
 731         # the closing bracket is an optional paren
 732         and rhs.closing_bracket.type == token.RPAR
 733         and not rhs.closing_bracket.value
 734         # it's not an import (optional parens are the only thing we can split on
 735         # in this case; attempting a split without them is a waste of time)
 736         and not line.is_import
 737         # there are no standalone comments in the body
 738         and not rhs.body.contains_standalone_comments(0)
 739         # and we can actually remove the parens
 740         and can_omit_invisible_parens(rhs, mode.line_length)
 741     ):
 742         omit = {id(rhs.closing_bracket), *omit}
 743         try:
 744             # The RHSResult Omitting Optional Parens.
 745             rhs_oop = _first_right_hand_split(line, omit=omit)
 746             if not (
 747                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 748                 # the split is right after `=`
 749                 and len(rhs.head.leaves) >= 2
 750                 and rhs.head.leaves[-2].type == token.EQUAL
 751                 # the left side of assignment contains brackets
 752                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 753                 # the left side of assignment is short enough (the -1 is for the ending
 754                 # optional paren)
 755                 and is_line_short_enough(
 756                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 757                 )
 758                 # the left side of assignment won't explode further because of magic
 759                 # trailing comma
 760                 and rhs.head.magic_trailing_comma is None
 761                 # the split by omitting optional parens isn't preferred by some other
 762                 # reason
 763                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 764             ):
 765                 yield from _maybe_split_omitting_optional_parens(
 766                     rhs_oop, line, mode, features=features, omit=omit
 767                 )
 768                 return
 769
 770         except CannotSplit as e:
 771             if not (
 772                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 773             ):
 774                 raise CannotSplit(
 775                     "Splitting failed, body is still too long and can't be split."
 776                 ) from e
 777
 778             elif (
 779                 rhs.head.contains_multiline_strings()
 780                 or rhs.tail.contains_multiline_strings()
 781             ):
 782                 raise CannotSplit(
 783                     "The current optional pair of parentheses is bound to fail to"
 784                     " satisfy the splitting algorithm because the head or the tail"
 785                     " contains multiline strings which by definition never fit one"
 786                     " line."
 787                 ) from e
 788
 789     ensure_visible(rhs.opening_bracket)
 790     ensure_visible(rhs.closing_bracket)
 791     for result in (rhs.head, rhs.body, rhs.tail):
 792         if result:
 793             yield result
 794
 795
 796 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 797     """
 798     Returns whether we should prefer the result from a split omitting optional parens.
 799     """
 800     has_closing_bracket_after_assign = False
 801     for leaf in reversed(rhs_oop.head.leaves):
 802         if leaf.type == token.EQUAL:
 803             break
 804         if leaf.type in CLOSING_BRACKETS:
 805             has_closing_bracket_after_assign = True
 806             break
 807     return (
 808         # contains matching brackets after the `=` (done by checking there is a
 809         # closing bracket)
 810         has_closing_bracket_after_assign
 811         or (
 812             # the split is actually from inside the optional parens (done by checking
 813             # the first line still contains the `=`)
 814             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 815             # the first line is short enough
 816             and is_line_short_enough(rhs_oop.head, mode=mode)
 817         )
 818         # contains unsplittable type ignore
 819         or rhs_oop.head.contains_unsplittable_type_ignore()
 820         or rhs_oop.body.contains_unsplittable_type_ignore()
 821         or rhs_oop.tail.contains_unsplittable_type_ignore()
 822     )
 823
 824
 825 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 826     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 827
 828     Do nothing otherwise.
 829
 830     A left- or right-hand split is based on a pair of brackets. Content before
 831     (and including) the opening bracket is left on one line, content inside the
 832     brackets is put on a separate line, and finally content starting with and
 833     following the closing bracket is put on a separate line.
 834
 835     Those are called `head`, `body`, and `tail`, respectively. If the split
 836     produced the same line (all content in `head`) or ended up with an empty `body`
 837     and the `tail` is just the closing bracket, then it's considered failed.
 838     """
 839     tail_len = len(str(tail).strip())
 840     if not body:
 841         if tail_len == 0:
 842             raise CannotSplit("Splitting brackets produced the same line")
 843
 844         elif tail_len < 3:
 845             raise CannotSplit(
 846                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 847                 " not worth it"
 848             )
 849
 850
 851 def bracket_split_build_line(
 852     leaves: List[Leaf],
 853     original: Line,
 854     opening_bracket: Leaf,
 855     *,
 856     component: _BracketSplitComponent,
 857 ) -> Line:
 858     """Return a new line with given `leaves` and respective comments from `original`.
 859
 860     If it's the head component, brackets will be tracked so trailing commas are
 861     respected.
 862
 863     If it's the body component, the result line is one-indented inside brackets and as
 864     such has its first leaf's prefix normalized and a trailing comma added when
 865     expected.
 866     """
 867     result = Line(mode=original.mode, depth=original.depth)
 868     if component is _BracketSplitComponent.body:
 869         result.inside_brackets = True
 870         result.depth += 1
 871         if leaves:
 872             # Since body is a new indent level, remove spurious leading whitespace.
 873             normalize_prefix(leaves[0], inside_brackets=True)
 874             # Ensure a trailing comma for imports and standalone function arguments, but
 875             # be careful not to add one after any comments or within type annotations.
 876             no_commas = (
 877                 original.is_def
 878                 and opening_bracket.value == "("
 879                 and not any(leaf.type == token.COMMA for leaf in leaves)
 880                 # In particular, don't add one within a parenthesized return annotation.
 881                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 882                 # be defined directly in the parent node, the parent of the parent ...
 883                 # and so on depending on how complex the return annotation is.
 884                 # This isn't perfect and there's some false negatives but they are in
 885                 # contexts were a comma is actually fine.
 886                 and not any(
 887                     node.prev_sibling.type == RARROW
 888                     for node in (
 889                         leaves[0].parent,
 890                         getattr(leaves[0].parent, "parent", None),
 891                     )
 892                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 893                 )
 894             )
 895
 896             if original.is_import or no_commas:
 897                 for i in range(len(leaves) - 1, -1, -1):
 898                     if leaves[i].type == STANDALONE_COMMENT:
 899                         continue
 900
 901                     if leaves[i].type != token.COMMA:
 902                         new_comma = Leaf(token.COMMA, ",")
 903                         leaves.insert(i + 1, new_comma)
 904                     break
 905
 906     leaves_to_track: Set[LeafID] = set()
 907     if component is _BracketSplitComponent.head:
 908         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
 909     # Populate the line
 910     for leaf in leaves:
 911         result.append(
 912             leaf,
 913             preformatted=True,
 914             track_bracket=id(leaf) in leaves_to_track,
 915         )
 916         for comment_after in original.comments_after(leaf):
 917             result.append(comment_after, preformatted=True)
 918     if component is _BracketSplitComponent.body and should_split_line(
 919         result, opening_bracket
 920     ):
 921         result.should_split_rhs = True
 922     return result
 923
 924
 925 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 926     """Normalize prefix of the first leaf in every line returned by `split_func`.
 927
 928     This is a decorator over relevant split functions.
 929     """
 930
 931     @wraps(split_func)
 932     def split_wrapper(
 933         line: Line, features: Collection[Feature], mode: Mode
 934     ) -> Iterator[Line]:
 935         for split_line in split_func(line, features, mode):
 936             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 937             yield split_line
 938
 939     return split_wrapper
 940
 941
 942 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
 943     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
 944         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
 945             return leaf_idx
 946     return None
 947
 948
 949 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
 950     if (
 951         safe
 952         and delimiter_priority == COMMA_PRIORITY
 953         and line.leaves[-1].type != token.COMMA
 954         and line.leaves[-1].type != STANDALONE_COMMENT
 955     ):
 956         new_comma = Leaf(token.COMMA, ",")
 957         line.append(new_comma)
 958     return line
 959
 960
 961 @dont_increase_indentation
 962 def delimiter_split(
 963     line: Line, features: Collection[Feature], mode: Mode
 964 ) -> Iterator[Line]:
 965     """Split according to delimiters of the highest priority.
 966
 967     If the appropriate Features are given, the split will add trailing commas
 968     also in function signatures and calls that contain `*` and `**`.
 969     """
 970     try:
 971         last_leaf = line.leaves[-1]
 972     except IndexError:
 973         raise CannotSplit("Line empty") from None
 974
 975     bt = line.bracket_tracker
 976     try:
 977         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 978     except ValueError:
 979         raise CannotSplit("No delimiters found") from None
 980
 981     if delimiter_priority == DOT_PRIORITY:
 982         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 983             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 984
 985     current_line = Line(
 986         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 987     )
 988     lowest_depth = sys.maxsize
 989     trailing_comma_safe = True
 990
 991     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 992         """Append `leaf` to current line or to new line if appending impossible."""
 993         nonlocal current_line
 994         try:
 995             current_line.append_safe(leaf, preformatted=True)
 996         except ValueError:
 997             yield current_line
 998
 999             current_line = Line(
1000                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1001             )
1002             current_line.append(leaf)
1003
1004     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1005     for leaf_idx, leaf in enumerate(line.leaves):
1006         yield from append_to_line(leaf)
1007
1008         for comment_after in line.comments_after(leaf):
1009             yield from append_to_line(comment_after)
1010
1011         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1012         if leaf.bracket_depth == lowest_depth:
1013             if is_vararg(leaf, within={syms.typedargslist}):
1014                 trailing_comma_safe = (
1015                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1016                 )
1017             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1018                 trailing_comma_safe = (
1019                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1020                 )
1021
1022         if (
1023             Preview.add_trailing_comma_consistently in mode
1024             and last_leaf.type == STANDALONE_COMMENT
1025             and leaf_idx == last_non_comment_leaf
1026         ):
1027             current_line = _safe_add_trailing_comma(
1028                 trailing_comma_safe, delimiter_priority, current_line
1029             )
1030
1031         leaf_priority = bt.delimiters.get(id(leaf))
1032         if leaf_priority == delimiter_priority:
1033             yield current_line
1034
1035             current_line = Line(
1036                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1037             )
1038     if current_line:
1039         current_line = _safe_add_trailing_comma(
1040             trailing_comma_safe, delimiter_priority, current_line
1041         )
1042         yield current_line
1043
1044
1045 @dont_increase_indentation
1046 def standalone_comment_split(
1047     line: Line, features: Collection[Feature], mode: Mode
1048 ) -> Iterator[Line]:
1049     """Split standalone comments from the rest of the line."""
1050     if not line.contains_standalone_comments(0):
1051         raise CannotSplit("Line does not have any standalone comments")
1052
1053     current_line = Line(
1054         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1055     )
1056
1057     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1058         """Append `leaf` to current line or to new line if appending impossible."""
1059         nonlocal current_line
1060         try:
1061             current_line.append_safe(leaf, preformatted=True)
1062         except ValueError:
1063             yield current_line
1064
1065             current_line = Line(
1066                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1067             )
1068             current_line.append(leaf)
1069
1070     for leaf in line.leaves:
1071         yield from append_to_line(leaf)
1072
1073         for comment_after in line.comments_after(leaf):
1074             yield from append_to_line(comment_after)
1075
1076     if current_line:
1077         yield current_line
1078
1079
1080 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1081     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1082     else.
1083
1084     Note: don't use backslashes for formatting or you'll lose your voting rights.
1085     """
1086     if not inside_brackets:
1087         spl = leaf.prefix.split("#")
1088         if "\\" not in spl[0]:
1089             nl_count = spl[-1].count("\n")
1090             if len(spl) > 1:
1091                 nl_count -= 1
1092             leaf.prefix = "\n" * nl_count
1093             return
1094
1095     leaf.prefix = ""
1096
1097
1098 def normalize_invisible_parens(
1099     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1100 ) -> None:
1101     """Make existing optional parentheses invisible or create new ones.
1102
1103     `parens_after` is a set of string leaf values immediately after which parens
1104     should be put.
1105
1106     Standardizes on visible parentheses for single-element tuples, and keeps
1107     existing visible parentheses for other tuples and generator expressions.
1108     """
1109     for pc in list_comments(node.prefix, is_endmarker=False):
1110         if pc.value in FMT_OFF:
1111             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1112             return
1113
1114     # The multiple context managers grammar has a different pattern, thus this is
1115     # separate from the for-loop below. This possibly wraps them in invisible parens,
1116     # and later will be removed in remove_with_parens when needed.
1117     if node.type == syms.with_stmt:
1118         _maybe_wrap_cms_in_parens(node, mode, features)
1119
1120     check_lpar = False
1121     for index, child in enumerate(list(node.children)):
1122         # Fixes a bug where invisible parens are not properly stripped from
1123         # assignment statements that contain type annotations.
1124         if isinstance(child, Node) and child.type == syms.annassign:
1125             normalize_invisible_parens(
1126                 child, parens_after=parens_after, mode=mode, features=features
1127             )
1128
1129         # Add parentheses around long tuple unpacking in assignments.
1130         if (
1131             index == 0
1132             and isinstance(child, Node)
1133             and child.type == syms.testlist_star_expr
1134         ):
1135             check_lpar = True
1136
1137         if check_lpar:
1138             if (
1139                 child.type == syms.atom
1140                 and node.type == syms.for_stmt
1141                 and isinstance(child.prev_sibling, Leaf)
1142                 and child.prev_sibling.type == token.NAME
1143                 and child.prev_sibling.value == "for"
1144             ):
1145                 if maybe_make_parens_invisible_in_atom(
1146                     child,
1147                     parent=node,
1148                     remove_brackets_around_comma=True,
1149                 ):
1150                     wrap_in_parentheses(node, child, visible=False)
1151             elif isinstance(child, Node) and node.type == syms.with_stmt:
1152                 remove_with_parens(child, node)
1153             elif child.type == syms.atom:
1154                 if maybe_make_parens_invisible_in_atom(
1155                     child,
1156                     parent=node,
1157                 ):
1158                     wrap_in_parentheses(node, child, visible=False)
1159             elif is_one_tuple(child):
1160                 wrap_in_parentheses(node, child, visible=True)
1161             elif node.type == syms.import_from:
1162                 _normalize_import_from(node, child, index)
1163                 break
1164             elif (
1165                 index == 1
1166                 and child.type == token.STAR
1167                 and node.type == syms.except_clause
1168             ):
1169                 # In except* (PEP 654), the star is actually part of
1170                 # of the keyword. So we need to skip the insertion of
1171                 # invisible parentheses to work more precisely.
1172                 continue
1173
1174             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1175                 wrap_in_parentheses(node, child, visible=False)
1176
1177         comma_check = child.type == token.COMMA
1178
1179         check_lpar = isinstance(child, Leaf) and (
1180             child.value in parens_after or comma_check
1181         )
1182
1183
1184 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1185     # "import from" nodes store parentheses directly as part of
1186     # the statement
1187     if is_lpar_token(child):
1188         assert is_rpar_token(parent.children[-1])
1189         # make parentheses invisible
1190         child.value = ""
1191         parent.children[-1].value = ""
1192     elif child.type != token.STAR:
1193         # insert invisible parentheses
1194         parent.insert_child(index, Leaf(token.LPAR, ""))
1195         parent.append_child(Leaf(token.RPAR, ""))
1196
1197
1198 def remove_await_parens(node: Node) -> None:
1199     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1200         if (
1201             node.children[1].type == syms.atom
1202             and node.children[1].children[0].type == token.LPAR
1203         ):
1204             if maybe_make_parens_invisible_in_atom(
1205                 node.children[1],
1206                 parent=node,
1207                 remove_brackets_around_comma=True,
1208             ):
1209                 wrap_in_parentheses(node, node.children[1], visible=False)
1210
1211             # Since await is an expression we shouldn't remove
1212             # brackets in cases where this would change
1213             # the AST due to operator precedence.
1214             # Therefore we only aim to remove brackets around
1215             # power nodes that aren't also await expressions themselves.
1216             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1217             # N.B. We've still removed any redundant nested brackets though :)
1218             opening_bracket = cast(Leaf, node.children[1].children[0])
1219             closing_bracket = cast(Leaf, node.children[1].children[-1])
1220             bracket_contents = node.children[1].children[1]
1221             if isinstance(bracket_contents, Node):
1222                 if bracket_contents.type != syms.power:
1223                     ensure_visible(opening_bracket)
1224                     ensure_visible(closing_bracket)
1225                 elif (
1226                     bracket_contents.type == syms.power
1227                     and bracket_contents.children[0].type == token.AWAIT
1228                 ):
1229                     ensure_visible(opening_bracket)
1230                     ensure_visible(closing_bracket)
1231                     # If we are in a nested await then recurse down.
1232                     remove_await_parens(bracket_contents)
1233
1234
1235 def _maybe_wrap_cms_in_parens(
1236     node: Node, mode: Mode, features: Collection[Feature]
1237 ) -> None:
1238     """When enabled and safe, wrap the multiple context managers in invisible parens.
1239
1240     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1241     """
1242     if (
1243         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1244         or Preview.wrap_multiple_context_managers_in_parens not in mode
1245         or len(node.children) <= 2
1246         # If it's an atom, it's already wrapped in parens.
1247         or node.children[1].type == syms.atom
1248     ):
1249         return
1250     colon_index: Optional[int] = None
1251     for i in range(2, len(node.children)):
1252         if node.children[i].type == token.COLON:
1253             colon_index = i
1254             break
1255     if colon_index is not None:
1256         lpar = Leaf(token.LPAR, "")
1257         rpar = Leaf(token.RPAR, "")
1258         context_managers = node.children[1:colon_index]
1259         for child in context_managers:
1260             child.remove()
1261         # After wrapping, the with_stmt will look like this:
1262         #   with_stmt
1263         #     NAME 'with'
1264         #     atom
1265         #       LPAR ''
1266         #       testlist_gexp
1267         #         ... <-- context_managers
1268         #       /testlist_gexp
1269         #       RPAR ''
1270         #     /atom
1271         #     COLON ':'
1272         new_child = Node(
1273             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1274         )
1275         node.insert_child(1, new_child)
1276
1277
1278 def remove_with_parens(node: Node, parent: Node) -> None:
1279     """Recursively hide optional parens in `with` statements."""
1280     # Removing all unnecessary parentheses in with statements in one pass is a tad
1281     # complex as different variations of bracketed statements result in pretty
1282     # different parse trees:
1283     #
1284     # with (open("file")) as f:                       # this is an asexpr_test
1285     #     ...
1286     #
1287     # with (open("file") as f):                       # this is an atom containing an
1288     #     ...                                         # asexpr_test
1289     #
1290     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1291     #     ...                                         # asexpr_test
1292     #
1293     # with (open("file") as f, open("file") as f):    # an atom containing a
1294     #     ...                                         # testlist_gexp which then
1295     #                                                 # contains multiple asexpr_test(s)
1296     if node.type == syms.atom:
1297         if maybe_make_parens_invisible_in_atom(
1298             node,
1299             parent=parent,
1300             remove_brackets_around_comma=True,
1301         ):
1302             wrap_in_parentheses(parent, node, visible=False)
1303         if isinstance(node.children[1], Node):
1304             remove_with_parens(node.children[1], node)
1305     elif node.type == syms.testlist_gexp:
1306         for child in node.children:
1307             if isinstance(child, Node):
1308                 remove_with_parens(child, node)
1309     elif node.type == syms.asexpr_test and not any(
1310         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1311     ):
1312         if maybe_make_parens_invisible_in_atom(
1313             node.children[0],
1314             parent=node,
1315             remove_brackets_around_comma=True,
1316         ):
1317             wrap_in_parentheses(node, node.children[0], visible=False)
1318
1319
1320 def maybe_make_parens_invisible_in_atom(
1321     node: LN,
1322     parent: LN,
1323     remove_brackets_around_comma: bool = False,
1324 ) -> bool:
1325     """If it's safe, make the parens in the atom `node` invisible, recursively.
1326     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1327     as they are redundant.
1328
1329     Returns whether the node should itself be wrapped in invisible parentheses.
1330     """
1331     if (
1332         node.type != syms.atom
1333         or is_empty_tuple(node)
1334         or is_one_tuple(node)
1335         or (is_yield(node) and parent.type != syms.expr_stmt)
1336         or (
1337             # This condition tries to prevent removing non-optional brackets
1338             # around a tuple, however, can be a bit overzealous so we provide
1339             # and option to skip this check for `for` and `with` statements.
1340             not remove_brackets_around_comma
1341             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1342         )
1343         or is_tuple_containing_walrus(node)
1344     ):
1345         return False
1346
1347     if is_walrus_assignment(node):
1348         if parent.type in [
1349             syms.annassign,
1350             syms.expr_stmt,
1351             syms.assert_stmt,
1352             syms.return_stmt,
1353             syms.except_clause,
1354             syms.funcdef,
1355             syms.with_stmt,
1356             # these ones aren't useful to end users, but they do please fuzzers
1357             syms.for_stmt,
1358             syms.del_stmt,
1359             syms.for_stmt,
1360         ]:
1361             return False
1362
1363     first = node.children[0]
1364     last = node.children[-1]
1365     if is_lpar_token(first) and is_rpar_token(last):
1366         middle = node.children[1]
1367         # make parentheses invisible
1368         first.value = ""
1369         last.value = ""
1370         maybe_make_parens_invisible_in_atom(
1371             middle,
1372             parent=parent,
1373             remove_brackets_around_comma=remove_brackets_around_comma,
1374         )
1375
1376         if is_atom_with_invisible_parens(middle):
1377             # Strip the invisible parens from `middle` by replacing
1378             # it with the child in-between the invisible parens
1379             middle.replace(middle.children[1])
1380
1381         return False
1382
1383     return True
1384
1385
1386 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1387     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1388
1389     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1390         return False
1391
1392     # We're essentially checking if the body is delimited by commas and there's more
1393     # than one of them (we're excluding the trailing comma and if the delimiter priority
1394     # is still commas, that means there's more).
1395     exclude = set()
1396     trailing_comma = False
1397     try:
1398         last_leaf = line.leaves[-1]
1399         if last_leaf.type == token.COMMA:
1400             trailing_comma = True
1401             exclude.add(id(last_leaf))
1402         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1403     except (IndexError, ValueError):
1404         return False
1405
1406     return max_priority == COMMA_PRIORITY and (
1407         (line.mode.magic_trailing_comma and trailing_comma)
1408         # always explode imports
1409         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1410     )
1411
1412
1413 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1414     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1415
1416     Brackets can be omitted if the entire trailer up to and including
1417     a preceding closing bracket fits in one line.
1418
1419     Yielded sets are cumulative (contain results of previous yields, too).  First
1420     set is empty, unless the line should explode, in which case bracket pairs until
1421     the one that needs to explode are omitted.
1422     """
1423
1424     omit: Set[LeafID] = set()
1425     if not line.magic_trailing_comma:
1426         yield omit
1427
1428     length = 4 * line.depth
1429     opening_bracket: Optional[Leaf] = None
1430     closing_bracket: Optional[Leaf] = None
1431     inner_brackets: Set[LeafID] = set()
1432     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1433         length += leaf_length
1434         if length > line_length:
1435             break
1436
1437         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1438         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1439             break
1440
1441         if opening_bracket:
1442             if leaf is opening_bracket:
1443                 opening_bracket = None
1444             elif leaf.type in CLOSING_BRACKETS:
1445                 prev = line.leaves[index - 1] if index > 0 else None
1446                 if (
1447                     prev
1448                     and prev.type == token.COMMA
1449                     and leaf.opening_bracket is not None
1450                     and not is_one_sequence_between(
1451                         leaf.opening_bracket, leaf, line.leaves
1452                     )
1453                 ):
1454                     # Never omit bracket pairs with trailing commas.
1455                     # We need to explode on those.
1456                     break
1457
1458                 inner_brackets.add(id(leaf))
1459         elif leaf.type in CLOSING_BRACKETS:
1460             prev = line.leaves[index - 1] if index > 0 else None
1461             if prev and prev.type in OPENING_BRACKETS:
1462                 # Empty brackets would fail a split so treat them as "inner"
1463                 # brackets (e.g. only add them to the `omit` set if another
1464                 # pair of brackets was good enough.
1465                 inner_brackets.add(id(leaf))
1466                 continue
1467
1468             if closing_bracket:
1469                 omit.add(id(closing_bracket))
1470                 omit.update(inner_brackets)
1471                 inner_brackets.clear()
1472                 yield omit
1473
1474             if (
1475                 prev
1476                 and prev.type == token.COMMA
1477                 and leaf.opening_bracket is not None
1478                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1479             ):
1480                 # Never omit bracket pairs with trailing commas.
1481                 # We need to explode on those.
1482                 break
1483
1484             if leaf.value:
1485                 opening_bracket = leaf.opening_bracket
1486                 closing_bracket = leaf
1487
1488
1489 def run_transformer(
1490     line: Line,
1491     transform: Transformer,
1492     mode: Mode,
1493     features: Collection[Feature],
1494     *,
1495     line_str: str = "",
1496 ) -> List[Line]:
1497     if not line_str:
1498         line_str = line_to_string(line)
1499     result: List[Line] = []
1500     for transformed_line in transform(line, features, mode):
1501         if str(transformed_line).strip("\n") == line_str:
1502             raise CannotTransform("Line transformer returned an unchanged result")
1503
1504         result.extend(transform_line(transformed_line, mode=mode, features=features))
1505
1506     features_set = set(features)
1507     if (
1508         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1509         or transform.__class__.__name__ != "rhs"
1510         or not line.bracket_tracker.invisible
1511         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1512         or line.contains_multiline_strings()
1513         or result[0].contains_uncollapsable_type_comments()
1514         or result[0].contains_unsplittable_type_ignore()
1515         or is_line_short_enough(result[0], mode=mode)
1516         # If any leaves have no parents (which _can_ occur since
1517         # `transform(line)` potentially destroys the line's underlying node
1518         # structure), then we can't proceed. Doing so would cause the below
1519         # call to `append_leaves()` to fail.
1520         or any(leaf.parent is None for leaf in line.leaves)
1521     ):
1522         return result
1523
1524     line_copy = line.clone()
1525     append_leaves(line_copy, line, line.leaves)
1526     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1527     second_opinion = run_transformer(
1528         line_copy, transform, mode, features_fop, line_str=line_str
1529     )
1530     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1531         result = second_opinion
1532     return result