src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import replace
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     RHSResult,
  20     append_leaves,
  21     can_be_split,
  22     can_omit_invisible_parens,
  23     is_line_short_enough,
  24     line_to_string,
  25 )
  26 from black.mode import Feature, Mode, Preview
  27 from black.nodes import (
  28     ASSIGNMENTS,
  29     BRACKETS,
  30     CLOSING_BRACKETS,
  31     OPENING_BRACKETS,
  32     RARROW,
  33     STANDALONE_COMMENT,
  34     STATEMENT,
  35     WHITESPACE,
  36     Visitor,
  37     ensure_visible,
  38     is_arith_like,
  39     is_async_stmt_or_funcdef,
  40     is_atom_with_invisible_parens,
  41     is_docstring,
  42     is_empty_tuple,
  43     is_lpar_token,
  44     is_multiline_string,
  45     is_name_token,
  46     is_one_sequence_between,
  47     is_one_tuple,
  48     is_rpar_token,
  49     is_stub_body,
  50     is_stub_suite,
  51     is_tuple_containing_walrus,
  52     is_vararg,
  53     is_walrus_assignment,
  54     is_yield,
  55     syms,
  56     wrap_in_parentheses,
  57 )
  58 from black.numerics import normalize_numeric_literal
  59 from black.strings import (
  60     fix_docstring,
  61     get_string_prefix,
  62     normalize_string_prefix,
  63     normalize_string_quotes,
  64     normalize_unicode_escape_sequences,
  65 )
  66 from black.trans import (
  67     CannotTransform,
  68     StringMerger,
  69     StringParenStripper,
  70     StringParenWrapper,
  71     StringSplitter,
  72     Transformer,
  73     hug_power_op,
  74 )
  75 from blib2to3.pgen2 import token
  76 from blib2to3.pytree import Leaf, Node
  77
  78 # types
  79 LeafID = int
  80 LN = Union[Leaf, Node]
  81
  82
  83 class CannotSplit(CannotTransform):
  84     """A readable split that fits the allotted line length is impossible."""
  85
  86
  87 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  88 # See also https://github.com/mypyc/mypyc/issues/827.
  89 class LineGenerator(Visitor[Line]):
  90     """Generates reformatted Line objects.  Empty lines are not emitted.
  91
  92     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  93     in ways that will no longer stringify to valid Python code on the tree.
  94     """
  95
  96     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  97         self.mode = mode
  98         self.features = features
  99         self.current_line: Line
 100         self.__post_init__()
 101
 102     def line(self, indent: int = 0) -> Iterator[Line]:
 103         """Generate a line.
 104
 105         If the line is empty, only emit if it makes sense.
 106         If the line is too long, split it first and then generate.
 107
 108         If any lines were generated, set up a new current_line.
 109         """
 110         if not self.current_line:
 111             self.current_line.depth += indent
 112             return  # Line is empty, don't emit. Creating a new one unnecessary.
 113
 114         if (
 115             Preview.improved_async_statements_handling in self.mode
 116             and len(self.current_line.leaves) == 1
 117             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
 118         ):
 119             # Special case for async def/for/with statements. `visit_async_stmt`
 120             # adds an `ASYNC` leaf then visits the child def/for/with statement
 121             # nodes. Line yields from those nodes shouldn't treat the former
 122             # `ASYNC` leaf as a complete line.
 123             return
 124
 125         complete_line = self.current_line
 126         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 127         yield complete_line
 128
 129     def visit_default(self, node: LN) -> Iterator[Line]:
 130         """Default `visit_*()` implementation. Recurses to children of `node`."""
 131         if isinstance(node, Leaf):
 132             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 133             for comment in generate_comments(node):
 134                 if any_open_brackets:
 135                     # any comment within brackets is subject to splitting
 136                     self.current_line.append(comment)
 137                 elif comment.type == token.COMMENT:
 138                     # regular trailing comment
 139                     self.current_line.append(comment)
 140                     yield from self.line()
 141
 142                 else:
 143                     # regular standalone comment
 144                     yield from self.line()
 145
 146                     self.current_line.append(comment)
 147                     yield from self.line()
 148
 149             normalize_prefix(node, inside_brackets=any_open_brackets)
 150             if self.mode.string_normalization and node.type == token.STRING:
 151                 node.value = normalize_string_prefix(node.value)
 152                 node.value = normalize_string_quotes(node.value)
 153             if node.type == token.NUMBER:
 154                 normalize_numeric_literal(node)
 155             if node.type not in WHITESPACE:
 156                 self.current_line.append(node)
 157         yield from super().visit_default(node)
 158
 159     def visit_test(self, node: Node) -> Iterator[Line]:
 160         """Visit an `x if y else z` test"""
 161
 162         if Preview.parenthesize_conditional_expressions in self.mode:
 163             already_parenthesized = (
 164                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 165             )
 166
 167             if not already_parenthesized:
 168                 lpar = Leaf(token.LPAR, "")
 169                 rpar = Leaf(token.RPAR, "")
 170                 node.insert_child(0, lpar)
 171                 node.append_child(rpar)
 172
 173         yield from self.visit_default(node)
 174
 175     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 176         """Increase indentation level, maybe yield a line."""
 177         # In blib2to3 INDENT never holds comments.
 178         yield from self.line(+1)
 179         yield from self.visit_default(node)
 180
 181     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 182         """Decrease indentation level, maybe yield a line."""
 183         # The current line might still wait for trailing comments.  At DEDENT time
 184         # there won't be any (they would be prefixes on the preceding NEWLINE).
 185         # Emit the line then.
 186         yield from self.line()
 187
 188         # While DEDENT has no value, its prefix may contain standalone comments
 189         # that belong to the current indentation level.  Get 'em.
 190         yield from self.visit_default(node)
 191
 192         # Finally, emit the dedent.
 193         yield from self.line(-1)
 194
 195     def visit_stmt(
 196         self, node: Node, keywords: Set[str], parens: Set[str]
 197     ) -> Iterator[Line]:
 198         """Visit a statement.
 199
 200         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 201         `def`, `with`, `class`, `assert`, and assignments.
 202
 203         The relevant Python language `keywords` for a given statement will be
 204         NAME leaves within it. This methods puts those on a separate line.
 205
 206         `parens` holds a set of string leaf values immediately after which
 207         invisible parens should be put.
 208         """
 209         normalize_invisible_parens(
 210             node, parens_after=parens, mode=self.mode, features=self.features
 211         )
 212         for child in node.children:
 213             if is_name_token(child) and child.value in keywords:
 214                 yield from self.line()
 215
 216             yield from self.visit(child)
 217
 218     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 219         if Preview.wrap_long_dict_values_in_parens in self.mode:
 220             for i, child in enumerate(node.children):
 221                 if i == 0:
 222                     continue
 223                 if node.children[i - 1].type == token.COLON:
 224                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 225                         if maybe_make_parens_invisible_in_atom(
 226                             child,
 227                             parent=node,
 228                             remove_brackets_around_comma=False,
 229                         ):
 230                             wrap_in_parentheses(node, child, visible=False)
 231                     else:
 232                         wrap_in_parentheses(node, child, visible=False)
 233         yield from self.visit_default(node)
 234
 235     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 236         """Visit function definition."""
 237         yield from self.line()
 238
 239         # Remove redundant brackets around return type annotation.
 240         is_return_annotation = False
 241         for child in node.children:
 242             if child.type == token.RARROW:
 243                 is_return_annotation = True
 244             elif is_return_annotation:
 245                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 246                     if maybe_make_parens_invisible_in_atom(
 247                         child,
 248                         parent=node,
 249                         remove_brackets_around_comma=False,
 250                     ):
 251                         wrap_in_parentheses(node, child, visible=False)
 252                 else:
 253                     wrap_in_parentheses(node, child, visible=False)
 254                 is_return_annotation = False
 255
 256         for child in node.children:
 257             yield from self.visit(child)
 258
 259     def visit_match_case(self, node: Node) -> Iterator[Line]:
 260         """Visit either a match or case statement."""
 261         normalize_invisible_parens(
 262             node, parens_after=set(), mode=self.mode, features=self.features
 263         )
 264
 265         yield from self.line()
 266         for child in node.children:
 267             yield from self.visit(child)
 268
 269     def visit_suite(self, node: Node) -> Iterator[Line]:
 270         """Visit a suite."""
 271         if self.mode.is_pyi and is_stub_suite(node):
 272             yield from self.visit(node.children[2])
 273         else:
 274             yield from self.visit_default(node)
 275
 276     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 277         """Visit a statement without nested statements."""
 278         prev_type: Optional[int] = None
 279         for child in node.children:
 280             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 281                 wrap_in_parentheses(node, child, visible=False)
 282             prev_type = child.type
 283
 284         is_suite_like = node.parent and node.parent.type in STATEMENT
 285         if is_suite_like:
 286             if self.mode.is_pyi and is_stub_body(node):
 287                 yield from self.visit_default(node)
 288             else:
 289                 yield from self.line(+1)
 290                 yield from self.visit_default(node)
 291                 yield from self.line(-1)
 292
 293         else:
 294             if (
 295                 not self.mode.is_pyi
 296                 or not node.parent
 297                 or not is_stub_suite(node.parent)
 298             ):
 299                 yield from self.line()
 300             yield from self.visit_default(node)
 301
 302     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 303         """Visit `async def`, `async for`, `async with`."""
 304         yield from self.line()
 305
 306         children = iter(node.children)
 307         for child in children:
 308             yield from self.visit(child)
 309
 310             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 311                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 312                 # line.
 313                 break
 314
 315         internal_stmt = next(children)
 316         if Preview.improved_async_statements_handling in self.mode:
 317             yield from self.visit(internal_stmt)
 318         else:
 319             for child in internal_stmt.children:
 320                 yield from self.visit(child)
 321
 322     def visit_decorators(self, node: Node) -> Iterator[Line]:
 323         """Visit decorators."""
 324         for child in node.children:
 325             yield from self.line()
 326             yield from self.visit(child)
 327
 328     def visit_power(self, node: Node) -> Iterator[Line]:
 329         for idx, leaf in enumerate(node.children[:-1]):
 330             next_leaf = node.children[idx + 1]
 331
 332             if not isinstance(leaf, Leaf):
 333                 continue
 334
 335             value = leaf.value.lower()
 336             if (
 337                 leaf.type == token.NUMBER
 338                 and next_leaf.type == syms.trailer
 339                 # Ensure that we are in an attribute trailer
 340                 and next_leaf.children[0].type == token.DOT
 341                 # It shouldn't wrap hexadecimal, binary and octal literals
 342                 and not value.startswith(("0x", "0b", "0o"))
 343                 # It shouldn't wrap complex literals
 344                 and "j" not in value
 345             ):
 346                 wrap_in_parentheses(node, leaf)
 347
 348         remove_await_parens(node)
 349
 350         yield from self.visit_default(node)
 351
 352     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 353         """Remove a semicolon and put the other statement on a separate line."""
 354         yield from self.line()
 355
 356     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 357         """End of file. Process outstanding comments and end with a newline."""
 358         yield from self.visit_default(leaf)
 359         yield from self.line()
 360
 361     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 362         if not self.current_line.bracket_tracker.any_open_brackets():
 363             yield from self.line()
 364         yield from self.visit_default(leaf)
 365
 366     def visit_factor(self, node: Node) -> Iterator[Line]:
 367         """Force parentheses between a unary op and a binary power:
 368
 369         -2 ** 8 -> -(2 ** 8)
 370         """
 371         _operator, operand = node.children
 372         if (
 373             operand.type == syms.power
 374             and len(operand.children) == 3
 375             and operand.children[1].type == token.DOUBLESTAR
 376         ):
 377             lpar = Leaf(token.LPAR, "(")
 378             rpar = Leaf(token.RPAR, ")")
 379             index = operand.remove() or 0
 380             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 381         yield from self.visit_default(node)
 382
 383     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 384         if Preview.hex_codes_in_unicode_sequences in self.mode:
 385             normalize_unicode_escape_sequences(leaf)
 386
 387         if is_docstring(leaf) and "\\\n" not in leaf.value:
 388             # We're ignoring docstrings with backslash newline escapes because changing
 389             # indentation of those changes the AST representation of the code.
 390             if self.mode.string_normalization:
 391                 docstring = normalize_string_prefix(leaf.value)
 392                 # visit_default() does handle string normalization for us, but
 393                 # since this method acts differently depending on quote style (ex.
 394                 # see padding logic below), there's a possibility for unstable
 395                 # formatting as visit_default() is called *after*. To avoid a
 396                 # situation where this function formats a docstring differently on
 397                 # the second pass, normalize it early.
 398                 docstring = normalize_string_quotes(docstring)
 399             else:
 400                 docstring = leaf.value
 401             prefix = get_string_prefix(docstring)
 402             docstring = docstring[len(prefix) :]  # Remove the prefix
 403             quote_char = docstring[0]
 404             # A natural way to remove the outer quotes is to do:
 405             #   docstring = docstring.strip(quote_char)
 406             # but that breaks on """""x""" (which is '""x').
 407             # So we actually need to remove the first character and the next two
 408             # characters but only if they are the same as the first.
 409             quote_len = 1 if docstring[1] != quote_char else 3
 410             docstring = docstring[quote_len:-quote_len]
 411             docstring_started_empty = not docstring
 412             indent = " " * 4 * self.current_line.depth
 413
 414             if is_multiline_string(leaf):
 415                 docstring = fix_docstring(docstring, indent)
 416             else:
 417                 docstring = docstring.strip()
 418
 419             has_trailing_backslash = False
 420             if docstring:
 421                 # Add some padding if the docstring starts / ends with a quote mark.
 422                 if docstring[0] == quote_char:
 423                     docstring = " " + docstring
 424                 if docstring[-1] == quote_char:
 425                     docstring += " "
 426                 if docstring[-1] == "\\":
 427                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 428                     if backslash_count % 2:
 429                         # Odd number of tailing backslashes, add some padding to
 430                         # avoid escaping the closing string quote.
 431                         docstring += " "
 432                         has_trailing_backslash = True
 433             elif not docstring_started_empty:
 434                 docstring = " "
 435
 436             # We could enforce triple quotes at this point.
 437             quote = quote_char * quote_len
 438
 439             # It's invalid to put closing single-character quotes on a new line.
 440             if self.mode and quote_len == 3:
 441                 # We need to find the length of the last line of the docstring
 442                 # to find if we can add the closing quotes to the line without
 443                 # exceeding the maximum line length.
 444                 # If docstring is one line, we don't put the closing quotes on a
 445                 # separate line because it looks ugly (#3320).
 446                 lines = docstring.splitlines()
 447                 last_line_length = len(lines[-1]) if docstring else 0
 448
 449                 # If adding closing quotes would cause the last line to exceed
 450                 # the maximum line length then put a line break before the
 451                 # closing quotes
 452                 if (
 453                     len(lines) > 1
 454                     and last_line_length + quote_len > self.mode.line_length
 455                     and len(indent) + quote_len <= self.mode.line_length
 456                     and not has_trailing_backslash
 457                 ):
 458                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 459                 else:
 460                     leaf.value = prefix + quote + docstring + quote
 461             else:
 462                 leaf.value = prefix + quote + docstring + quote
 463
 464         yield from self.visit_default(leaf)
 465
 466     def __post_init__(self) -> None:
 467         """You are in a twisty little maze of passages."""
 468         self.current_line = Line(mode=self.mode)
 469
 470         v = self.visit_stmt
 471         Ø: Set[str] = set()
 472         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 473         self.visit_if_stmt = partial(
 474             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 475         )
 476         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 477         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 478         self.visit_try_stmt = partial(
 479             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 480         )
 481         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 482         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 483         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 484         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 485         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 486         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 487         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 488         self.visit_async_funcdef = self.visit_async_stmt
 489         self.visit_decorated = self.visit_decorators
 490
 491         # PEP 634
 492         self.visit_match_stmt = self.visit_match_case
 493         self.visit_case_block = self.visit_match_case
 494
 495
 496 def transform_line(
 497     line: Line, mode: Mode, features: Collection[Feature] = ()
 498 ) -> Iterator[Line]:
 499     """Transform a `line`, potentially splitting it into many lines.
 500
 501     They should fit in the allotted `line_length` but might not be able to.
 502
 503     `features` are syntactical features that may be used in the output.
 504     """
 505     if line.is_comment:
 506         yield line
 507         return
 508
 509     line_str = line_to_string(line)
 510
 511     ll = mode.line_length
 512     sn = mode.string_normalization
 513     string_merge = StringMerger(ll, sn)
 514     string_paren_strip = StringParenStripper(ll, sn)
 515     string_split = StringSplitter(ll, sn)
 516     string_paren_wrap = StringParenWrapper(ll, sn)
 517
 518     transformers: List[Transformer]
 519     if (
 520         not line.contains_uncollapsable_type_comments()
 521         and not line.should_split_rhs
 522         and not line.magic_trailing_comma
 523         and (
 524             is_line_short_enough(line, mode=mode, line_str=line_str)
 525             or line.contains_unsplittable_type_ignore()
 526         )
 527         and not (line.inside_brackets and line.contains_standalone_comments())
 528     ):
 529         # Only apply basic string preprocessing, since lines shouldn't be split here.
 530         if Preview.string_processing in mode:
 531             transformers = [string_merge, string_paren_strip]
 532         else:
 533             transformers = []
 534     elif line.is_def:
 535         transformers = [left_hand_split]
 536     else:
 537
 538         def _rhs(
 539             self: object, line: Line, features: Collection[Feature], mode: Mode
 540         ) -> Iterator[Line]:
 541             """Wraps calls to `right_hand_split`.
 542
 543             The calls increasingly `omit` right-hand trailers (bracket pairs with
 544             content), meaning the trailers get glued together to split on another
 545             bracket pair instead.
 546             """
 547             for omit in generate_trailers_to_omit(line, mode.line_length):
 548                 lines = list(right_hand_split(line, mode, features, omit=omit))
 549                 # Note: this check is only able to figure out if the first line of the
 550                 # *current* transformation fits in the line length.  This is true only
 551                 # for simple cases.  All others require running more transforms via
 552                 # `transform_line()`.  This check doesn't know if those would succeed.
 553                 if is_line_short_enough(lines[0], mode=mode):
 554                     yield from lines
 555                     return
 556
 557             # All splits failed, best effort split with no omits.
 558             # This mostly happens to multiline strings that are by definition
 559             # reported as not fitting a single line, as well as lines that contain
 560             # trailing commas (those have to be exploded).
 561             yield from right_hand_split(line, mode, features=features)
 562
 563         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 564         # __name__ attribute which is needed in `run_transformer` further down.
 565         # Unfortunately a nested class breaks mypyc too. So a class must be created
 566         # via type ... https://github.com/mypyc/mypyc/issues/884
 567         rhs = type("rhs", (), {"__call__": _rhs})()
 568
 569         if Preview.string_processing in mode:
 570             if line.inside_brackets:
 571                 transformers = [
 572                     string_merge,
 573                     string_paren_strip,
 574                     string_split,
 575                     delimiter_split,
 576                     standalone_comment_split,
 577                     string_paren_wrap,
 578                     rhs,
 579                 ]
 580             else:
 581                 transformers = [
 582                     string_merge,
 583                     string_paren_strip,
 584                     string_split,
 585                     string_paren_wrap,
 586                     rhs,
 587                 ]
 588         else:
 589             if line.inside_brackets:
 590                 transformers = [delimiter_split, standalone_comment_split, rhs]
 591             else:
 592                 transformers = [rhs]
 593     # It's always safe to attempt hugging of power operations and pretty much every line
 594     # could match.
 595     transformers.append(hug_power_op)
 596
 597     for transform in transformers:
 598         # We are accumulating lines in `result` because we might want to abort
 599         # mission and return the original line in the end, or attempt a different
 600         # split altogether.
 601         try:
 602             result = run_transformer(line, transform, mode, features, line_str=line_str)
 603         except CannotTransform:
 604             continue
 605         else:
 606             yield from result
 607             break
 608
 609     else:
 610         yield line
 611
 612
 613 class _BracketSplitComponent(Enum):
 614     head = auto()
 615     body = auto()
 616     tail = auto()
 617
 618
 619 def left_hand_split(
 620     line: Line, _features: Collection[Feature], mode: Mode
 621 ) -> Iterator[Line]:
 622     """Split line into many lines, starting with the first matching bracket pair.
 623
 624     Note: this usually looks weird, only use this for function definitions.
 625     Prefer RHS otherwise.  This is why this function is not symmetrical with
 626     :func:`right_hand_split` which also handles optional parentheses.
 627     """
 628     tail_leaves: List[Leaf] = []
 629     body_leaves: List[Leaf] = []
 630     head_leaves: List[Leaf] = []
 631     current_leaves = head_leaves
 632     matching_bracket: Optional[Leaf] = None
 633     for leaf in line.leaves:
 634         if (
 635             current_leaves is body_leaves
 636             and leaf.type in CLOSING_BRACKETS
 637             and leaf.opening_bracket is matching_bracket
 638             and isinstance(matching_bracket, Leaf)
 639         ):
 640             ensure_visible(leaf)
 641             ensure_visible(matching_bracket)
 642             current_leaves = tail_leaves if body_leaves else head_leaves
 643         current_leaves.append(leaf)
 644         if current_leaves is head_leaves:
 645             if leaf.type in OPENING_BRACKETS:
 646                 matching_bracket = leaf
 647                 current_leaves = body_leaves
 648     if not matching_bracket:
 649         raise CannotSplit("No brackets found")
 650
 651     head = bracket_split_build_line(
 652         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 653     )
 654     body = bracket_split_build_line(
 655         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 656     )
 657     tail = bracket_split_build_line(
 658         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 659     )
 660     bracket_split_succeeded_or_raise(head, body, tail)
 661     for result in (head, body, tail):
 662         if result:
 663             yield result
 664
 665
 666 def right_hand_split(
 667     line: Line,
 668     mode: Mode,
 669     features: Collection[Feature] = (),
 670     omit: Collection[LeafID] = (),
 671 ) -> Iterator[Line]:
 672     """Split line into many lines, starting with the last matching bracket pair.
 673
 674     If the split was by optional parentheses, attempt splitting without them, too.
 675     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 676     this split.
 677
 678     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 679     """
 680     rhs_result = _first_right_hand_split(line, omit=omit)
 681     yield from _maybe_split_omitting_optional_parens(
 682         rhs_result, line, mode, features=features, omit=omit
 683     )
 684
 685
 686 def _first_right_hand_split(
 687     line: Line,
 688     omit: Collection[LeafID] = (),
 689 ) -> RHSResult:
 690     """Split the line into head, body, tail starting with the last bracket pair.
 691
 692     Note: this function should not have side effects. It's relied upon by
 693     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 694     splitting on the right side of an assignment statement.
 695     """
 696     tail_leaves: List[Leaf] = []
 697     body_leaves: List[Leaf] = []
 698     head_leaves: List[Leaf] = []
 699     current_leaves = tail_leaves
 700     opening_bracket: Optional[Leaf] = None
 701     closing_bracket: Optional[Leaf] = None
 702     for leaf in reversed(line.leaves):
 703         if current_leaves is body_leaves:
 704             if leaf is opening_bracket:
 705                 current_leaves = head_leaves if body_leaves else tail_leaves
 706         current_leaves.append(leaf)
 707         if current_leaves is tail_leaves:
 708             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 709                 opening_bracket = leaf.opening_bracket
 710                 closing_bracket = leaf
 711                 current_leaves = body_leaves
 712     if not (opening_bracket and closing_bracket and head_leaves):
 713         # If there is no opening or closing_bracket that means the split failed and
 714         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 715         # the matching `opening_bracket` wasn't available on `line` anymore.
 716         raise CannotSplit("No brackets found")
 717
 718     tail_leaves.reverse()
 719     body_leaves.reverse()
 720     head_leaves.reverse()
 721     head = bracket_split_build_line(
 722         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 723     )
 724     body = bracket_split_build_line(
 725         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 726     )
 727     tail = bracket_split_build_line(
 728         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 729     )
 730     bracket_split_succeeded_or_raise(head, body, tail)
 731     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 732
 733
 734 def _maybe_split_omitting_optional_parens(
 735     rhs: RHSResult,
 736     line: Line,
 737     mode: Mode,
 738     features: Collection[Feature] = (),
 739     omit: Collection[LeafID] = (),
 740 ) -> Iterator[Line]:
 741     if (
 742         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 743         # the opening bracket is an optional paren
 744         and rhs.opening_bracket.type == token.LPAR
 745         and not rhs.opening_bracket.value
 746         # the closing bracket is an optional paren
 747         and rhs.closing_bracket.type == token.RPAR
 748         and not rhs.closing_bracket.value
 749         # it's not an import (optional parens are the only thing we can split on
 750         # in this case; attempting a split without them is a waste of time)
 751         and not line.is_import
 752         # there are no standalone comments in the body
 753         and not rhs.body.contains_standalone_comments(0)
 754         # and we can actually remove the parens
 755         and can_omit_invisible_parens(rhs, mode.line_length)
 756     ):
 757         omit = {id(rhs.closing_bracket), *omit}
 758         try:
 759             # The RHSResult Omitting Optional Parens.
 760             rhs_oop = _first_right_hand_split(line, omit=omit)
 761             if not (
 762                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 763                 # the split is right after `=`
 764                 and len(rhs.head.leaves) >= 2
 765                 and rhs.head.leaves[-2].type == token.EQUAL
 766                 # the left side of assignment contains brackets
 767                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 768                 # the left side of assignment is short enough (the -1 is for the ending
 769                 # optional paren)
 770                 and is_line_short_enough(
 771                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 772                 )
 773                 # the left side of assignment won't explode further because of magic
 774                 # trailing comma
 775                 and rhs.head.magic_trailing_comma is None
 776                 # the split by omitting optional parens isn't preferred by some other
 777                 # reason
 778                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 779             ):
 780                 yield from _maybe_split_omitting_optional_parens(
 781                     rhs_oop, line, mode, features=features, omit=omit
 782                 )
 783                 return
 784
 785         except CannotSplit as e:
 786             if not (
 787                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 788             ):
 789                 raise CannotSplit(
 790                     "Splitting failed, body is still too long and can't be split."
 791                 ) from e
 792
 793             elif (
 794                 rhs.head.contains_multiline_strings()
 795                 or rhs.tail.contains_multiline_strings()
 796             ):
 797                 raise CannotSplit(
 798                     "The current optional pair of parentheses is bound to fail to"
 799                     " satisfy the splitting algorithm because the head or the tail"
 800                     " contains multiline strings which by definition never fit one"
 801                     " line."
 802                 ) from e
 803
 804     ensure_visible(rhs.opening_bracket)
 805     ensure_visible(rhs.closing_bracket)
 806     for result in (rhs.head, rhs.body, rhs.tail):
 807         if result:
 808             yield result
 809
 810
 811 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 812     """
 813     Returns whether we should prefer the result from a split omitting optional parens.
 814     """
 815     has_closing_bracket_after_assign = False
 816     for leaf in reversed(rhs_oop.head.leaves):
 817         if leaf.type == token.EQUAL:
 818             break
 819         if leaf.type in CLOSING_BRACKETS:
 820             has_closing_bracket_after_assign = True
 821             break
 822     return (
 823         # contains matching brackets after the `=` (done by checking there is a
 824         # closing bracket)
 825         has_closing_bracket_after_assign
 826         or (
 827             # the split is actually from inside the optional parens (done by checking
 828             # the first line still contains the `=`)
 829             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 830             # the first line is short enough
 831             and is_line_short_enough(rhs_oop.head, mode=mode)
 832         )
 833         # contains unsplittable type ignore
 834         or rhs_oop.head.contains_unsplittable_type_ignore()
 835         or rhs_oop.body.contains_unsplittable_type_ignore()
 836         or rhs_oop.tail.contains_unsplittable_type_ignore()
 837     )
 838
 839
 840 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 841     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 842
 843     Do nothing otherwise.
 844
 845     A left- or right-hand split is based on a pair of brackets. Content before
 846     (and including) the opening bracket is left on one line, content inside the
 847     brackets is put on a separate line, and finally content starting with and
 848     following the closing bracket is put on a separate line.
 849
 850     Those are called `head`, `body`, and `tail`, respectively. If the split
 851     produced the same line (all content in `head`) or ended up with an empty `body`
 852     and the `tail` is just the closing bracket, then it's considered failed.
 853     """
 854     tail_len = len(str(tail).strip())
 855     if not body:
 856         if tail_len == 0:
 857             raise CannotSplit("Splitting brackets produced the same line")
 858
 859         elif tail_len < 3:
 860             raise CannotSplit(
 861                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 862                 " not worth it"
 863             )
 864
 865
 866 def bracket_split_build_line(
 867     leaves: List[Leaf],
 868     original: Line,
 869     opening_bracket: Leaf,
 870     *,
 871     component: _BracketSplitComponent,
 872 ) -> Line:
 873     """Return a new line with given `leaves` and respective comments from `original`.
 874
 875     If it's the head component, brackets will be tracked so trailing commas are
 876     respected.
 877
 878     If it's the body component, the result line is one-indented inside brackets and as
 879     such has its first leaf's prefix normalized and a trailing comma added when
 880     expected.
 881     """
 882     result = Line(mode=original.mode, depth=original.depth)
 883     if component is _BracketSplitComponent.body:
 884         result.inside_brackets = True
 885         result.depth += 1
 886         if leaves:
 887             # Since body is a new indent level, remove spurious leading whitespace.
 888             normalize_prefix(leaves[0], inside_brackets=True)
 889             # Ensure a trailing comma for imports and standalone function arguments, but
 890             # be careful not to add one after any comments or within type annotations.
 891             no_commas = (
 892                 original.is_def
 893                 and opening_bracket.value == "("
 894                 and not any(leaf.type == token.COMMA for leaf in leaves)
 895                 # In particular, don't add one within a parenthesized return annotation.
 896                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 897                 # be defined directly in the parent node, the parent of the parent ...
 898                 # and so on depending on how complex the return annotation is.
 899                 # This isn't perfect and there's some false negatives but they are in
 900                 # contexts were a comma is actually fine.
 901                 and not any(
 902                     node.prev_sibling.type == RARROW
 903                     for node in (
 904                         leaves[0].parent,
 905                         getattr(leaves[0].parent, "parent", None),
 906                     )
 907                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 908                 )
 909             )
 910
 911             if original.is_import or no_commas:
 912                 for i in range(len(leaves) - 1, -1, -1):
 913                     if leaves[i].type == STANDALONE_COMMENT:
 914                         continue
 915
 916                     if leaves[i].type != token.COMMA:
 917                         new_comma = Leaf(token.COMMA, ",")
 918                         leaves.insert(i + 1, new_comma)
 919                     break
 920
 921     leaves_to_track: Set[LeafID] = set()
 922     if component is _BracketSplitComponent.head:
 923         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
 924     # Populate the line
 925     for leaf in leaves:
 926         result.append(
 927             leaf,
 928             preformatted=True,
 929             track_bracket=id(leaf) in leaves_to_track,
 930         )
 931         for comment_after in original.comments_after(leaf):
 932             result.append(comment_after, preformatted=True)
 933     if component is _BracketSplitComponent.body and should_split_line(
 934         result, opening_bracket
 935     ):
 936         result.should_split_rhs = True
 937     return result
 938
 939
 940 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 941     """Normalize prefix of the first leaf in every line returned by `split_func`.
 942
 943     This is a decorator over relevant split functions.
 944     """
 945
 946     @wraps(split_func)
 947     def split_wrapper(
 948         line: Line, features: Collection[Feature], mode: Mode
 949     ) -> Iterator[Line]:
 950         for split_line in split_func(line, features, mode):
 951             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 952             yield split_line
 953
 954     return split_wrapper
 955
 956
 957 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
 958     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
 959         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
 960             return leaf_idx
 961     return None
 962
 963
 964 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
 965     if (
 966         safe
 967         and delimiter_priority == COMMA_PRIORITY
 968         and line.leaves[-1].type != token.COMMA
 969         and line.leaves[-1].type != STANDALONE_COMMENT
 970     ):
 971         new_comma = Leaf(token.COMMA, ",")
 972         line.append(new_comma)
 973     return line
 974
 975
 976 @dont_increase_indentation
 977 def delimiter_split(
 978     line: Line, features: Collection[Feature], mode: Mode
 979 ) -> Iterator[Line]:
 980     """Split according to delimiters of the highest priority.
 981
 982     If the appropriate Features are given, the split will add trailing commas
 983     also in function signatures and calls that contain `*` and `**`.
 984     """
 985     try:
 986         last_leaf = line.leaves[-1]
 987     except IndexError:
 988         raise CannotSplit("Line empty") from None
 989
 990     bt = line.bracket_tracker
 991     try:
 992         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 993     except ValueError:
 994         raise CannotSplit("No delimiters found") from None
 995
 996     if delimiter_priority == DOT_PRIORITY:
 997         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 998             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 999
1000     current_line = Line(
1001         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1002     )
1003     lowest_depth = sys.maxsize
1004     trailing_comma_safe = True
1005
1006     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1007         """Append `leaf` to current line or to new line if appending impossible."""
1008         nonlocal current_line
1009         try:
1010             current_line.append_safe(leaf, preformatted=True)
1011         except ValueError:
1012             yield current_line
1013
1014             current_line = Line(
1015                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1016             )
1017             current_line.append(leaf)
1018
1019     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1020     for leaf_idx, leaf in enumerate(line.leaves):
1021         yield from append_to_line(leaf)
1022
1023         for comment_after in line.comments_after(leaf):
1024             yield from append_to_line(comment_after)
1025
1026         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1027         if leaf.bracket_depth == lowest_depth:
1028             if is_vararg(leaf, within={syms.typedargslist}):
1029                 trailing_comma_safe = (
1030                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1031                 )
1032             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1033                 trailing_comma_safe = (
1034                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1035                 )
1036
1037         if (
1038             Preview.add_trailing_comma_consistently in mode
1039             and last_leaf.type == STANDALONE_COMMENT
1040             and leaf_idx == last_non_comment_leaf
1041         ):
1042             current_line = _safe_add_trailing_comma(
1043                 trailing_comma_safe, delimiter_priority, current_line
1044             )
1045
1046         leaf_priority = bt.delimiters.get(id(leaf))
1047         if leaf_priority == delimiter_priority:
1048             yield current_line
1049
1050             current_line = Line(
1051                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1052             )
1053     if current_line:
1054         current_line = _safe_add_trailing_comma(
1055             trailing_comma_safe, delimiter_priority, current_line
1056         )
1057         yield current_line
1058
1059
1060 @dont_increase_indentation
1061 def standalone_comment_split(
1062     line: Line, features: Collection[Feature], mode: Mode
1063 ) -> Iterator[Line]:
1064     """Split standalone comments from the rest of the line."""
1065     if not line.contains_standalone_comments(0):
1066         raise CannotSplit("Line does not have any standalone comments")
1067
1068     current_line = Line(
1069         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1070     )
1071
1072     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1073         """Append `leaf` to current line or to new line if appending impossible."""
1074         nonlocal current_line
1075         try:
1076             current_line.append_safe(leaf, preformatted=True)
1077         except ValueError:
1078             yield current_line
1079
1080             current_line = Line(
1081                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1082             )
1083             current_line.append(leaf)
1084
1085     for leaf in line.leaves:
1086         yield from append_to_line(leaf)
1087
1088         for comment_after in line.comments_after(leaf):
1089             yield from append_to_line(comment_after)
1090
1091     if current_line:
1092         yield current_line
1093
1094
1095 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1096     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1097     else.
1098
1099     Note: don't use backslashes for formatting or you'll lose your voting rights.
1100     """
1101     if not inside_brackets:
1102         spl = leaf.prefix.split("#")
1103         if "\\" not in spl[0]:
1104             nl_count = spl[-1].count("\n")
1105             if len(spl) > 1:
1106                 nl_count -= 1
1107             leaf.prefix = "\n" * nl_count
1108             return
1109
1110     leaf.prefix = ""
1111
1112
1113 def normalize_invisible_parens(
1114     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1115 ) -> None:
1116     """Make existing optional parentheses invisible or create new ones.
1117
1118     `parens_after` is a set of string leaf values immediately after which parens
1119     should be put.
1120
1121     Standardizes on visible parentheses for single-element tuples, and keeps
1122     existing visible parentheses for other tuples and generator expressions.
1123     """
1124     for pc in list_comments(node.prefix, is_endmarker=False):
1125         if pc.value in FMT_OFF:
1126             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1127             return
1128
1129     # The multiple context managers grammar has a different pattern, thus this is
1130     # separate from the for-loop below. This possibly wraps them in invisible parens,
1131     # and later will be removed in remove_with_parens when needed.
1132     if node.type == syms.with_stmt:
1133         _maybe_wrap_cms_in_parens(node, mode, features)
1134
1135     check_lpar = False
1136     for index, child in enumerate(list(node.children)):
1137         # Fixes a bug where invisible parens are not properly stripped from
1138         # assignment statements that contain type annotations.
1139         if isinstance(child, Node) and child.type == syms.annassign:
1140             normalize_invisible_parens(
1141                 child, parens_after=parens_after, mode=mode, features=features
1142             )
1143
1144         # Add parentheses around long tuple unpacking in assignments.
1145         if (
1146             index == 0
1147             and isinstance(child, Node)
1148             and child.type == syms.testlist_star_expr
1149         ):
1150             check_lpar = True
1151
1152         if check_lpar:
1153             if (
1154                 child.type == syms.atom
1155                 and node.type == syms.for_stmt
1156                 and isinstance(child.prev_sibling, Leaf)
1157                 and child.prev_sibling.type == token.NAME
1158                 and child.prev_sibling.value == "for"
1159             ):
1160                 if maybe_make_parens_invisible_in_atom(
1161                     child,
1162                     parent=node,
1163                     remove_brackets_around_comma=True,
1164                 ):
1165                     wrap_in_parentheses(node, child, visible=False)
1166             elif isinstance(child, Node) and node.type == syms.with_stmt:
1167                 remove_with_parens(child, node)
1168             elif child.type == syms.atom:
1169                 if maybe_make_parens_invisible_in_atom(
1170                     child,
1171                     parent=node,
1172                 ):
1173                     wrap_in_parentheses(node, child, visible=False)
1174             elif is_one_tuple(child):
1175                 wrap_in_parentheses(node, child, visible=True)
1176             elif node.type == syms.import_from:
1177                 _normalize_import_from(node, child, index)
1178                 break
1179             elif (
1180                 index == 1
1181                 and child.type == token.STAR
1182                 and node.type == syms.except_clause
1183             ):
1184                 # In except* (PEP 654), the star is actually part of
1185                 # of the keyword. So we need to skip the insertion of
1186                 # invisible parentheses to work more precisely.
1187                 continue
1188
1189             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1190                 wrap_in_parentheses(node, child, visible=False)
1191
1192         comma_check = child.type == token.COMMA
1193
1194         check_lpar = isinstance(child, Leaf) and (
1195             child.value in parens_after or comma_check
1196         )
1197
1198
1199 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1200     # "import from" nodes store parentheses directly as part of
1201     # the statement
1202     if is_lpar_token(child):
1203         assert is_rpar_token(parent.children[-1])
1204         # make parentheses invisible
1205         child.value = ""
1206         parent.children[-1].value = ""
1207     elif child.type != token.STAR:
1208         # insert invisible parentheses
1209         parent.insert_child(index, Leaf(token.LPAR, ""))
1210         parent.append_child(Leaf(token.RPAR, ""))
1211
1212
1213 def remove_await_parens(node: Node) -> None:
1214     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1215         if (
1216             node.children[1].type == syms.atom
1217             and node.children[1].children[0].type == token.LPAR
1218         ):
1219             if maybe_make_parens_invisible_in_atom(
1220                 node.children[1],
1221                 parent=node,
1222                 remove_brackets_around_comma=True,
1223             ):
1224                 wrap_in_parentheses(node, node.children[1], visible=False)
1225
1226             # Since await is an expression we shouldn't remove
1227             # brackets in cases where this would change
1228             # the AST due to operator precedence.
1229             # Therefore we only aim to remove brackets around
1230             # power nodes that aren't also await expressions themselves.
1231             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1232             # N.B. We've still removed any redundant nested brackets though :)
1233             opening_bracket = cast(Leaf, node.children[1].children[0])
1234             closing_bracket = cast(Leaf, node.children[1].children[-1])
1235             bracket_contents = node.children[1].children[1]
1236             if isinstance(bracket_contents, Node):
1237                 if bracket_contents.type != syms.power:
1238                     ensure_visible(opening_bracket)
1239                     ensure_visible(closing_bracket)
1240                 elif (
1241                     bracket_contents.type == syms.power
1242                     and bracket_contents.children[0].type == token.AWAIT
1243                 ):
1244                     ensure_visible(opening_bracket)
1245                     ensure_visible(closing_bracket)
1246                     # If we are in a nested await then recurse down.
1247                     remove_await_parens(bracket_contents)
1248
1249
1250 def _maybe_wrap_cms_in_parens(
1251     node: Node, mode: Mode, features: Collection[Feature]
1252 ) -> None:
1253     """When enabled and safe, wrap the multiple context managers in invisible parens.
1254
1255     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1256     """
1257     if (
1258         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1259         or Preview.wrap_multiple_context_managers_in_parens not in mode
1260         or len(node.children) <= 2
1261         # If it's an atom, it's already wrapped in parens.
1262         or node.children[1].type == syms.atom
1263     ):
1264         return
1265     colon_index: Optional[int] = None
1266     for i in range(2, len(node.children)):
1267         if node.children[i].type == token.COLON:
1268             colon_index = i
1269             break
1270     if colon_index is not None:
1271         lpar = Leaf(token.LPAR, "")
1272         rpar = Leaf(token.RPAR, "")
1273         context_managers = node.children[1:colon_index]
1274         for child in context_managers:
1275             child.remove()
1276         # After wrapping, the with_stmt will look like this:
1277         #   with_stmt
1278         #     NAME 'with'
1279         #     atom
1280         #       LPAR ''
1281         #       testlist_gexp
1282         #         ... <-- context_managers
1283         #       /testlist_gexp
1284         #       RPAR ''
1285         #     /atom
1286         #     COLON ':'
1287         new_child = Node(
1288             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1289         )
1290         node.insert_child(1, new_child)
1291
1292
1293 def remove_with_parens(node: Node, parent: Node) -> None:
1294     """Recursively hide optional parens in `with` statements."""
1295     # Removing all unnecessary parentheses in with statements in one pass is a tad
1296     # complex as different variations of bracketed statements result in pretty
1297     # different parse trees:
1298     #
1299     # with (open("file")) as f:                       # this is an asexpr_test
1300     #     ...
1301     #
1302     # with (open("file") as f):                       # this is an atom containing an
1303     #     ...                                         # asexpr_test
1304     #
1305     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1306     #     ...                                         # asexpr_test
1307     #
1308     # with (open("file") as f, open("file") as f):    # an atom containing a
1309     #     ...                                         # testlist_gexp which then
1310     #                                                 # contains multiple asexpr_test(s)
1311     if node.type == syms.atom:
1312         if maybe_make_parens_invisible_in_atom(
1313             node,
1314             parent=parent,
1315             remove_brackets_around_comma=True,
1316         ):
1317             wrap_in_parentheses(parent, node, visible=False)
1318         if isinstance(node.children[1], Node):
1319             remove_with_parens(node.children[1], node)
1320     elif node.type == syms.testlist_gexp:
1321         for child in node.children:
1322             if isinstance(child, Node):
1323                 remove_with_parens(child, node)
1324     elif node.type == syms.asexpr_test and not any(
1325         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1326     ):
1327         if maybe_make_parens_invisible_in_atom(
1328             node.children[0],
1329             parent=node,
1330             remove_brackets_around_comma=True,
1331         ):
1332             wrap_in_parentheses(node, node.children[0], visible=False)
1333
1334
1335 def maybe_make_parens_invisible_in_atom(
1336     node: LN,
1337     parent: LN,
1338     remove_brackets_around_comma: bool = False,
1339 ) -> bool:
1340     """If it's safe, make the parens in the atom `node` invisible, recursively.
1341     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1342     as they are redundant.
1343
1344     Returns whether the node should itself be wrapped in invisible parentheses.
1345     """
1346     if (
1347         node.type != syms.atom
1348         or is_empty_tuple(node)
1349         or is_one_tuple(node)
1350         or (is_yield(node) and parent.type != syms.expr_stmt)
1351         or (
1352             # This condition tries to prevent removing non-optional brackets
1353             # around a tuple, however, can be a bit overzealous so we provide
1354             # and option to skip this check for `for` and `with` statements.
1355             not remove_brackets_around_comma
1356             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1357         )
1358         or is_tuple_containing_walrus(node)
1359     ):
1360         return False
1361
1362     if is_walrus_assignment(node):
1363         if parent.type in [
1364             syms.annassign,
1365             syms.expr_stmt,
1366             syms.assert_stmt,
1367             syms.return_stmt,
1368             syms.except_clause,
1369             syms.funcdef,
1370             syms.with_stmt,
1371             # these ones aren't useful to end users, but they do please fuzzers
1372             syms.for_stmt,
1373             syms.del_stmt,
1374             syms.for_stmt,
1375         ]:
1376             return False
1377
1378     first = node.children[0]
1379     last = node.children[-1]
1380     if is_lpar_token(first) and is_rpar_token(last):
1381         middle = node.children[1]
1382         # make parentheses invisible
1383         first.value = ""
1384         last.value = ""
1385         maybe_make_parens_invisible_in_atom(
1386             middle,
1387             parent=parent,
1388             remove_brackets_around_comma=remove_brackets_around_comma,
1389         )
1390
1391         if is_atom_with_invisible_parens(middle):
1392             # Strip the invisible parens from `middle` by replacing
1393             # it with the child in-between the invisible parens
1394             middle.replace(middle.children[1])
1395
1396         return False
1397
1398     return True
1399
1400
1401 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1402     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1403
1404     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1405         return False
1406
1407     # We're essentially checking if the body is delimited by commas and there's more
1408     # than one of them (we're excluding the trailing comma and if the delimiter priority
1409     # is still commas, that means there's more).
1410     exclude = set()
1411     trailing_comma = False
1412     try:
1413         last_leaf = line.leaves[-1]
1414         if last_leaf.type == token.COMMA:
1415             trailing_comma = True
1416             exclude.add(id(last_leaf))
1417         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1418     except (IndexError, ValueError):
1419         return False
1420
1421     return max_priority == COMMA_PRIORITY and (
1422         (line.mode.magic_trailing_comma and trailing_comma)
1423         # always explode imports
1424         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1425     )
1426
1427
1428 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1429     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1430
1431     Brackets can be omitted if the entire trailer up to and including
1432     a preceding closing bracket fits in one line.
1433
1434     Yielded sets are cumulative (contain results of previous yields, too).  First
1435     set is empty, unless the line should explode, in which case bracket pairs until
1436     the one that needs to explode are omitted.
1437     """
1438
1439     omit: Set[LeafID] = set()
1440     if not line.magic_trailing_comma:
1441         yield omit
1442
1443     length = 4 * line.depth
1444     opening_bracket: Optional[Leaf] = None
1445     closing_bracket: Optional[Leaf] = None
1446     inner_brackets: Set[LeafID] = set()
1447     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1448         length += leaf_length
1449         if length > line_length:
1450             break
1451
1452         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1453         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1454             break
1455
1456         if opening_bracket:
1457             if leaf is opening_bracket:
1458                 opening_bracket = None
1459             elif leaf.type in CLOSING_BRACKETS:
1460                 prev = line.leaves[index - 1] if index > 0 else None
1461                 if (
1462                     prev
1463                     and prev.type == token.COMMA
1464                     and leaf.opening_bracket is not None
1465                     and not is_one_sequence_between(
1466                         leaf.opening_bracket, leaf, line.leaves
1467                     )
1468                 ):
1469                     # Never omit bracket pairs with trailing commas.
1470                     # We need to explode on those.
1471                     break
1472
1473                 inner_brackets.add(id(leaf))
1474         elif leaf.type in CLOSING_BRACKETS:
1475             prev = line.leaves[index - 1] if index > 0 else None
1476             if prev and prev.type in OPENING_BRACKETS:
1477                 # Empty brackets would fail a split so treat them as "inner"
1478                 # brackets (e.g. only add them to the `omit` set if another
1479                 # pair of brackets was good enough.
1480                 inner_brackets.add(id(leaf))
1481                 continue
1482
1483             if closing_bracket:
1484                 omit.add(id(closing_bracket))
1485                 omit.update(inner_brackets)
1486                 inner_brackets.clear()
1487                 yield omit
1488
1489             if (
1490                 prev
1491                 and prev.type == token.COMMA
1492                 and leaf.opening_bracket is not None
1493                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1494             ):
1495                 # Never omit bracket pairs with trailing commas.
1496                 # We need to explode on those.
1497                 break
1498
1499             if leaf.value:
1500                 opening_bracket = leaf.opening_bracket
1501                 closing_bracket = leaf
1502
1503
1504 def run_transformer(
1505     line: Line,
1506     transform: Transformer,
1507     mode: Mode,
1508     features: Collection[Feature],
1509     *,
1510     line_str: str = "",
1511 ) -> List[Line]:
1512     if not line_str:
1513         line_str = line_to_string(line)
1514     result: List[Line] = []
1515     for transformed_line in transform(line, features, mode):
1516         if str(transformed_line).strip("\n") == line_str:
1517             raise CannotTransform("Line transformer returned an unchanged result")
1518
1519         result.extend(transform_line(transformed_line, mode=mode, features=features))
1520
1521     features_set = set(features)
1522     if (
1523         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1524         or transform.__class__.__name__ != "rhs"
1525         or not line.bracket_tracker.invisible
1526         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1527         or line.contains_multiline_strings()
1528         or result[0].contains_uncollapsable_type_comments()
1529         or result[0].contains_unsplittable_type_ignore()
1530         or is_line_short_enough(result[0], mode=mode)
1531         # If any leaves have no parents (which _can_ occur since
1532         # `transform(line)` potentially destroys the line's underlying node
1533         # structure), then we can't proceed. Doing so would cause the below
1534         # call to `append_leaves()` to fail.
1535         or any(leaf.parent is None for leaf in line.leaves)
1536     ):
1537         return result
1538
1539     line_copy = line.clone()
1540     append_leaves(line_copy, line, line.leaves)
1541     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1542     second_opinion = run_transformer(
1543         line_copy, transform, mode, features_fop, line_str=line_str
1544     )
1545     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1546         result = second_opinion
1547     return result