src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 import sys
   5 from dataclasses import dataclass
   6 from enum import Enum, auto
   7 from functools import partial, wraps
   8 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   9
  10 from black.brackets import (
  11     COMMA_PRIORITY,
  12     DOT_PRIORITY,
  13     get_leaves_inside_matching_brackets,
  14     max_delimiter_priority_in_atom,
  15 )
  16 from black.comments import FMT_OFF, generate_comments, list_comments
  17 from black.lines import (
  18     Line,
  19     append_leaves,
  20     can_be_split,
  21     can_omit_invisible_parens,
  22     is_line_short_enough,
  23     line_to_string,
  24 )
  25 from black.mode import Feature, Mode, Preview
  26 from black.nodes import (
  27     ASSIGNMENTS,
  28     BRACKETS,
  29     CLOSING_BRACKETS,
  30     OPENING_BRACKETS,
  31     RARROW,
  32     STANDALONE_COMMENT,
  33     STATEMENT,
  34     WHITESPACE,
  35     Visitor,
  36     ensure_visible,
  37     is_arith_like,
  38     is_atom_with_invisible_parens,
  39     is_docstring,
  40     is_empty_tuple,
  41     is_lpar_token,
  42     is_multiline_string,
  43     is_name_token,
  44     is_one_sequence_between,
  45     is_one_tuple,
  46     is_rpar_token,
  47     is_stub_body,
  48     is_stub_suite,
  49     is_vararg,
  50     is_walrus_assignment,
  51     is_yield,
  52     syms,
  53     wrap_in_parentheses,
  54 )
  55 from black.numerics import normalize_numeric_literal
  56 from black.strings import (
  57     fix_docstring,
  58     get_string_prefix,
  59     normalize_string_prefix,
  60     normalize_string_quotes,
  61 )
  62 from black.trans import (
  63     CannotTransform,
  64     StringMerger,
  65     StringParenStripper,
  66     StringParenWrapper,
  67     StringSplitter,
  68     Transformer,
  69     hug_power_op,
  70 )
  71 from blib2to3.pgen2 import token
  72 from blib2to3.pytree import Leaf, Node
  73
  74 # types
  75 LeafID = int
  76 LN = Union[Leaf, Node]
  77
  78
  79 class CannotSplit(CannotTransform):
  80     """A readable split that fits the allotted line length is impossible."""
  81
  82
  83 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  84 # See also https://github.com/mypyc/mypyc/issues/827.
  85 class LineGenerator(Visitor[Line]):
  86     """Generates reformatted Line objects.  Empty lines are not emitted.
  87
  88     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  89     in ways that will no longer stringify to valid Python code on the tree.
  90     """
  91
  92     def __init__(self, mode: Mode) -> None:
  93         self.mode = mode
  94         self.current_line: Line
  95         self.__post_init__()
  96
  97     def line(self, indent: int = 0) -> Iterator[Line]:
  98         """Generate a line.
  99
 100         If the line is empty, only emit if it makes sense.
 101         If the line is too long, split it first and then generate.
 102
 103         If any lines were generated, set up a new current_line.
 104         """
 105         if not self.current_line:
 106             self.current_line.depth += indent
 107             return  # Line is empty, don't emit. Creating a new one unnecessary.
 108
 109         complete_line = self.current_line
 110         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 111         yield complete_line
 112
 113     def visit_default(self, node: LN) -> Iterator[Line]:
 114         """Default `visit_*()` implementation. Recurses to children of `node`."""
 115         if isinstance(node, Leaf):
 116             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 117             for comment in generate_comments(node, preview=self.mode.preview):
 118                 if any_open_brackets:
 119                     # any comment within brackets is subject to splitting
 120                     self.current_line.append(comment)
 121                 elif comment.type == token.COMMENT:
 122                     # regular trailing comment
 123                     self.current_line.append(comment)
 124                     yield from self.line()
 125
 126                 else:
 127                     # regular standalone comment
 128                     yield from self.line()
 129
 130                     self.current_line.append(comment)
 131                     yield from self.line()
 132
 133             normalize_prefix(node, inside_brackets=any_open_brackets)
 134             if self.mode.string_normalization and node.type == token.STRING:
 135                 node.value = normalize_string_prefix(node.value)
 136                 node.value = normalize_string_quotes(node.value)
 137             if node.type == token.NUMBER:
 138                 normalize_numeric_literal(node)
 139             if node.type not in WHITESPACE:
 140                 self.current_line.append(node)
 141         yield from super().visit_default(node)
 142
 143     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 144         """Increase indentation level, maybe yield a line."""
 145         # In blib2to3 INDENT never holds comments.
 146         yield from self.line(+1)
 147         yield from self.visit_default(node)
 148
 149     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 150         """Decrease indentation level, maybe yield a line."""
 151         # The current line might still wait for trailing comments.  At DEDENT time
 152         # there won't be any (they would be prefixes on the preceding NEWLINE).
 153         # Emit the line then.
 154         yield from self.line()
 155
 156         # While DEDENT has no value, its prefix may contain standalone comments
 157         # that belong to the current indentation level.  Get 'em.
 158         yield from self.visit_default(node)
 159
 160         # Finally, emit the dedent.
 161         yield from self.line(-1)
 162
 163     def visit_stmt(
 164         self, node: Node, keywords: Set[str], parens: Set[str]
 165     ) -> Iterator[Line]:
 166         """Visit a statement.
 167
 168         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 169         `def`, `with`, `class`, `assert`, and assignments.
 170
 171         The relevant Python language `keywords` for a given statement will be
 172         NAME leaves within it. This methods puts those on a separate line.
 173
 174         `parens` holds a set of string leaf values immediately after which
 175         invisible parens should be put.
 176         """
 177         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
 178         for child in node.children:
 179             if is_name_token(child) and child.value in keywords:
 180                 yield from self.line()
 181
 182             yield from self.visit(child)
 183
 184     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 185         if Preview.wrap_long_dict_values_in_parens in self.mode:
 186             for i, child in enumerate(node.children):
 187                 if i == 0:
 188                     continue
 189                 if node.children[i - 1].type == token.COLON:
 190                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 191                         if maybe_make_parens_invisible_in_atom(
 192                             child,
 193                             parent=node,
 194                             remove_brackets_around_comma=False,
 195                         ):
 196                             wrap_in_parentheses(node, child, visible=False)
 197                     else:
 198                         wrap_in_parentheses(node, child, visible=False)
 199         yield from self.visit_default(node)
 200
 201     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 202         """Visit function definition."""
 203         if Preview.annotation_parens not in self.mode:
 204             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
 205         else:
 206             yield from self.line()
 207
 208             # Remove redundant brackets around return type annotation.
 209             is_return_annotation = False
 210             for child in node.children:
 211                 if child.type == token.RARROW:
 212                     is_return_annotation = True
 213                 elif is_return_annotation:
 214                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 215                         if maybe_make_parens_invisible_in_atom(
 216                             child,
 217                             parent=node,
 218                             remove_brackets_around_comma=False,
 219                         ):
 220                             wrap_in_parentheses(node, child, visible=False)
 221                     else:
 222                         wrap_in_parentheses(node, child, visible=False)
 223                     is_return_annotation = False
 224
 225             for child in node.children:
 226                 yield from self.visit(child)
 227
 228     def visit_match_case(self, node: Node) -> Iterator[Line]:
 229         """Visit either a match or case statement."""
 230         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
 231
 232         yield from self.line()
 233         for child in node.children:
 234             yield from self.visit(child)
 235
 236     def visit_suite(self, node: Node) -> Iterator[Line]:
 237         """Visit a suite."""
 238         if self.mode.is_pyi and is_stub_suite(node):
 239             yield from self.visit(node.children[2])
 240         else:
 241             yield from self.visit_default(node)
 242
 243     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 244         """Visit a statement without nested statements."""
 245         prev_type: Optional[int] = None
 246         for child in node.children:
 247             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 248                 wrap_in_parentheses(node, child, visible=False)
 249             prev_type = child.type
 250
 251         is_suite_like = node.parent and node.parent.type in STATEMENT
 252         if is_suite_like:
 253             if self.mode.is_pyi and is_stub_body(node):
 254                 yield from self.visit_default(node)
 255             else:
 256                 yield from self.line(+1)
 257                 yield from self.visit_default(node)
 258                 yield from self.line(-1)
 259
 260         else:
 261             if (
 262                 not self.mode.is_pyi
 263                 or not node.parent
 264                 or not is_stub_suite(node.parent)
 265             ):
 266                 yield from self.line()
 267             yield from self.visit_default(node)
 268
 269     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 270         """Visit `async def`, `async for`, `async with`."""
 271         yield from self.line()
 272
 273         children = iter(node.children)
 274         for child in children:
 275             yield from self.visit(child)
 276
 277             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 278                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 279                 # line.
 280                 break
 281
 282         internal_stmt = next(children)
 283         for child in internal_stmt.children:
 284             yield from self.visit(child)
 285
 286     def visit_decorators(self, node: Node) -> Iterator[Line]:
 287         """Visit decorators."""
 288         for child in node.children:
 289             yield from self.line()
 290             yield from self.visit(child)
 291
 292     def visit_power(self, node: Node) -> Iterator[Line]:
 293         for idx, leaf in enumerate(node.children[:-1]):
 294             next_leaf = node.children[idx + 1]
 295
 296             if not isinstance(leaf, Leaf):
 297                 continue
 298
 299             value = leaf.value.lower()
 300             if (
 301                 leaf.type == token.NUMBER
 302                 and next_leaf.type == syms.trailer
 303                 # Ensure that we are in an attribute trailer
 304                 and next_leaf.children[0].type == token.DOT
 305                 # It shouldn't wrap hexadecimal, binary and octal literals
 306                 and not value.startswith(("0x", "0b", "0o"))
 307                 # It shouldn't wrap complex literals
 308                 and "j" not in value
 309             ):
 310                 wrap_in_parentheses(node, leaf)
 311
 312         if Preview.remove_redundant_parens in self.mode:
 313             remove_await_parens(node)
 314
 315         yield from self.visit_default(node)
 316
 317     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 318         """Remove a semicolon and put the other statement on a separate line."""
 319         yield from self.line()
 320
 321     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 322         """End of file. Process outstanding comments and end with a newline."""
 323         yield from self.visit_default(leaf)
 324         yield from self.line()
 325
 326     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 327         if not self.current_line.bracket_tracker.any_open_brackets():
 328             yield from self.line()
 329         yield from self.visit_default(leaf)
 330
 331     def visit_factor(self, node: Node) -> Iterator[Line]:
 332         """Force parentheses between a unary op and a binary power:
 333
 334         -2 ** 8 -> -(2 ** 8)
 335         """
 336         _operator, operand = node.children
 337         if (
 338             operand.type == syms.power
 339             and len(operand.children) == 3
 340             and operand.children[1].type == token.DOUBLESTAR
 341         ):
 342             lpar = Leaf(token.LPAR, "(")
 343             rpar = Leaf(token.RPAR, ")")
 344             index = operand.remove() or 0
 345             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 346         yield from self.visit_default(node)
 347
 348     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 349         if is_docstring(leaf) and "\\\n" not in leaf.value:
 350             # We're ignoring docstrings with backslash newline escapes because changing
 351             # indentation of those changes the AST representation of the code.
 352             if Preview.normalize_docstring_quotes_and_prefixes_properly in self.mode:
 353                 # There was a bug where --skip-string-normalization wouldn't stop us
 354                 # from normalizing docstring prefixes. To maintain stability, we can
 355                 # only address this buggy behaviour while the preview style is enabled.
 356                 if self.mode.string_normalization:
 357                     docstring = normalize_string_prefix(leaf.value)
 358                     # visit_default() does handle string normalization for us, but
 359                     # since this method acts differently depending on quote style (ex.
 360                     # see padding logic below), there's a possibility for unstable
 361                     # formatting as visit_default() is called *after*. To avoid a
 362                     # situation where this function formats a docstring differently on
 363                     # the second pass, normalize it early.
 364                     docstring = normalize_string_quotes(docstring)
 365                 else:
 366                     docstring = leaf.value
 367             else:
 368                 # ... otherwise, we'll keep the buggy behaviour >.<
 369                 docstring = normalize_string_prefix(leaf.value)
 370             prefix = get_string_prefix(docstring)
 371             docstring = docstring[len(prefix) :]  # Remove the prefix
 372             quote_char = docstring[0]
 373             # A natural way to remove the outer quotes is to do:
 374             #   docstring = docstring.strip(quote_char)
 375             # but that breaks on """""x""" (which is '""x').
 376             # So we actually need to remove the first character and the next two
 377             # characters but only if they are the same as the first.
 378             quote_len = 1 if docstring[1] != quote_char else 3
 379             docstring = docstring[quote_len:-quote_len]
 380             docstring_started_empty = not docstring
 381             indent = " " * 4 * self.current_line.depth
 382
 383             if is_multiline_string(leaf):
 384                 docstring = fix_docstring(docstring, indent)
 385             else:
 386                 docstring = docstring.strip()
 387
 388             if docstring:
 389                 # Add some padding if the docstring starts / ends with a quote mark.
 390                 if docstring[0] == quote_char:
 391                     docstring = " " + docstring
 392                 if docstring[-1] == quote_char:
 393                     docstring += " "
 394                 if docstring[-1] == "\\":
 395                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 396                     if backslash_count % 2:
 397                         # Odd number of tailing backslashes, add some padding to
 398                         # avoid escaping the closing string quote.
 399                         docstring += " "
 400             elif not docstring_started_empty:
 401                 docstring = " "
 402
 403             # We could enforce triple quotes at this point.
 404             quote = quote_char * quote_len
 405
 406             # It's invalid to put closing single-character quotes on a new line.
 407             if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
 408                 # We need to find the length of the last line of the docstring
 409                 # to find if we can add the closing quotes to the line without
 410                 # exceeding the maximum line length.
 411                 # If docstring is one line, we don't put the closing quotes on a
 412                 # separate line because it looks ugly (#3320).
 413                 lines = docstring.splitlines()
 414                 last_line_length = len(lines[-1]) if docstring else 0
 415
 416                 # If adding closing quotes would cause the last line to exceed
 417                 # the maximum line length then put a line break before the
 418                 # closing quotes
 419                 if (
 420                     len(lines) > 1
 421                     and last_line_length + quote_len > self.mode.line_length
 422                 ):
 423                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 424                 else:
 425                     leaf.value = prefix + quote + docstring + quote
 426             else:
 427                 leaf.value = prefix + quote + docstring + quote
 428
 429         yield from self.visit_default(leaf)
 430
 431     def __post_init__(self) -> None:
 432         """You are in a twisty little maze of passages."""
 433         self.current_line = Line(mode=self.mode)
 434
 435         v = self.visit_stmt
 436         Ø: Set[str] = set()
 437         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 438         self.visit_if_stmt = partial(
 439             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 440         )
 441         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 442         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 443         self.visit_try_stmt = partial(
 444             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 445         )
 446         if self.mode.preview:
 447             self.visit_except_clause = partial(
 448                 v, keywords={"except"}, parens={"except"}
 449             )
 450             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 451         else:
 452             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
 453             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
 454         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 455         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 456         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 457         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 458         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 459         self.visit_async_funcdef = self.visit_async_stmt
 460         self.visit_decorated = self.visit_decorators
 461
 462         # PEP 634
 463         self.visit_match_stmt = self.visit_match_case
 464         self.visit_case_block = self.visit_match_case
 465
 466
 467 def transform_line(
 468     line: Line, mode: Mode, features: Collection[Feature] = ()
 469 ) -> Iterator[Line]:
 470     """Transform a `line`, potentially splitting it into many lines.
 471
 472     They should fit in the allotted `line_length` but might not be able to.
 473
 474     `features` are syntactical features that may be used in the output.
 475     """
 476     if line.is_comment:
 477         yield line
 478         return
 479
 480     line_str = line_to_string(line)
 481
 482     ll = mode.line_length
 483     sn = mode.string_normalization
 484     string_merge = StringMerger(ll, sn)
 485     string_paren_strip = StringParenStripper(ll, sn)
 486     string_split = StringSplitter(ll, sn)
 487     string_paren_wrap = StringParenWrapper(ll, sn)
 488
 489     transformers: List[Transformer]
 490     if (
 491         not line.contains_uncollapsable_type_comments()
 492         and not line.should_split_rhs
 493         and not line.magic_trailing_comma
 494         and (
 495             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
 496             or line.contains_unsplittable_type_ignore()
 497         )
 498         and not (line.inside_brackets and line.contains_standalone_comments())
 499     ):
 500         # Only apply basic string preprocessing, since lines shouldn't be split here.
 501         if Preview.string_processing in mode:
 502             transformers = [string_merge, string_paren_strip]
 503         else:
 504             transformers = []
 505     elif line.is_def:
 506         transformers = [left_hand_split]
 507     else:
 508
 509         def _rhs(
 510             self: object, line: Line, features: Collection[Feature]
 511         ) -> Iterator[Line]:
 512             """Wraps calls to `right_hand_split`.
 513
 514             The calls increasingly `omit` right-hand trailers (bracket pairs with
 515             content), meaning the trailers get glued together to split on another
 516             bracket pair instead.
 517             """
 518             for omit in generate_trailers_to_omit(line, mode.line_length):
 519                 lines = list(
 520                     right_hand_split(line, mode.line_length, features, omit=omit)
 521                 )
 522                 # Note: this check is only able to figure out if the first line of the
 523                 # *current* transformation fits in the line length.  This is true only
 524                 # for simple cases.  All others require running more transforms via
 525                 # `transform_line()`.  This check doesn't know if those would succeed.
 526                 if is_line_short_enough(lines[0], line_length=mode.line_length):
 527                     yield from lines
 528                     return
 529
 530             # All splits failed, best effort split with no omits.
 531             # This mostly happens to multiline strings that are by definition
 532             # reported as not fitting a single line, as well as lines that contain
 533             # trailing commas (those have to be exploded).
 534             yield from right_hand_split(
 535                 line, line_length=mode.line_length, features=features
 536             )
 537
 538         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 539         # __name__ attribute which is needed in `run_transformer` further down.
 540         # Unfortunately a nested class breaks mypyc too. So a class must be created
 541         # via type ... https://github.com/mypyc/mypyc/issues/884
 542         rhs = type("rhs", (), {"__call__": _rhs})()
 543
 544         if Preview.string_processing in mode:
 545             if line.inside_brackets:
 546                 transformers = [
 547                     string_merge,
 548                     string_paren_strip,
 549                     string_split,
 550                     delimiter_split,
 551                     standalone_comment_split,
 552                     string_paren_wrap,
 553                     rhs,
 554                 ]
 555             else:
 556                 transformers = [
 557                     string_merge,
 558                     string_paren_strip,
 559                     string_split,
 560                     string_paren_wrap,
 561                     rhs,
 562                 ]
 563         else:
 564             if line.inside_brackets:
 565                 transformers = [delimiter_split, standalone_comment_split, rhs]
 566             else:
 567                 transformers = [rhs]
 568     # It's always safe to attempt hugging of power operations and pretty much every line
 569     # could match.
 570     transformers.append(hug_power_op)
 571
 572     for transform in transformers:
 573         # We are accumulating lines in `result` because we might want to abort
 574         # mission and return the original line in the end, or attempt a different
 575         # split altogether.
 576         try:
 577             result = run_transformer(line, transform, mode, features, line_str=line_str)
 578         except CannotTransform:
 579             continue
 580         else:
 581             yield from result
 582             break
 583
 584     else:
 585         yield line
 586
 587
 588 class _BracketSplitComponent(Enum):
 589     head = auto()
 590     body = auto()
 591     tail = auto()
 592
 593
 594 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
 595     """Split line into many lines, starting with the first matching bracket pair.
 596
 597     Note: this usually looks weird, only use this for function definitions.
 598     Prefer RHS otherwise.  This is why this function is not symmetrical with
 599     :func:`right_hand_split` which also handles optional parentheses.
 600     """
 601     tail_leaves: List[Leaf] = []
 602     body_leaves: List[Leaf] = []
 603     head_leaves: List[Leaf] = []
 604     current_leaves = head_leaves
 605     matching_bracket: Optional[Leaf] = None
 606     for leaf in line.leaves:
 607         if (
 608             current_leaves is body_leaves
 609             and leaf.type in CLOSING_BRACKETS
 610             and leaf.opening_bracket is matching_bracket
 611             and isinstance(matching_bracket, Leaf)
 612         ):
 613             ensure_visible(leaf)
 614             ensure_visible(matching_bracket)
 615             current_leaves = tail_leaves if body_leaves else head_leaves
 616         current_leaves.append(leaf)
 617         if current_leaves is head_leaves:
 618             if leaf.type in OPENING_BRACKETS:
 619                 matching_bracket = leaf
 620                 current_leaves = body_leaves
 621     if not matching_bracket:
 622         raise CannotSplit("No brackets found")
 623
 624     head = bracket_split_build_line(
 625         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 626     )
 627     body = bracket_split_build_line(
 628         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 629     )
 630     tail = bracket_split_build_line(
 631         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 632     )
 633     bracket_split_succeeded_or_raise(head, body, tail)
 634     for result in (head, body, tail):
 635         if result:
 636             yield result
 637
 638
 639 @dataclass
 640 class _RHSResult:
 641     """Intermediate split result from a right hand split."""
 642
 643     head: Line
 644     body: Line
 645     tail: Line
 646     opening_bracket: Leaf
 647     closing_bracket: Leaf
 648
 649
 650 def right_hand_split(
 651     line: Line,
 652     line_length: int,
 653     features: Collection[Feature] = (),
 654     omit: Collection[LeafID] = (),
 655 ) -> Iterator[Line]:
 656     """Split line into many lines, starting with the last matching bracket pair.
 657
 658     If the split was by optional parentheses, attempt splitting without them, too.
 659     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 660     this split.
 661
 662     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 663     """
 664     rhs_result = _first_right_hand_split(line, omit=omit)
 665     yield from _maybe_split_omitting_optional_parens(
 666         rhs_result, line, line_length, features=features, omit=omit
 667     )
 668
 669
 670 def _first_right_hand_split(
 671     line: Line,
 672     omit: Collection[LeafID] = (),
 673 ) -> _RHSResult:
 674     """Split the line into head, body, tail starting with the last bracket pair.
 675
 676     Note: this function should not have side effects. It's relied upon by
 677     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 678     splitting on the right side of an assignment statement.
 679     """
 680     tail_leaves: List[Leaf] = []
 681     body_leaves: List[Leaf] = []
 682     head_leaves: List[Leaf] = []
 683     current_leaves = tail_leaves
 684     opening_bracket: Optional[Leaf] = None
 685     closing_bracket: Optional[Leaf] = None
 686     for leaf in reversed(line.leaves):
 687         if current_leaves is body_leaves:
 688             if leaf is opening_bracket:
 689                 current_leaves = head_leaves if body_leaves else tail_leaves
 690         current_leaves.append(leaf)
 691         if current_leaves is tail_leaves:
 692             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 693                 opening_bracket = leaf.opening_bracket
 694                 closing_bracket = leaf
 695                 current_leaves = body_leaves
 696     if not (opening_bracket and closing_bracket and head_leaves):
 697         # If there is no opening or closing_bracket that means the split failed and
 698         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 699         # the matching `opening_bracket` wasn't available on `line` anymore.
 700         raise CannotSplit("No brackets found")
 701
 702     tail_leaves.reverse()
 703     body_leaves.reverse()
 704     head_leaves.reverse()
 705     head = bracket_split_build_line(
 706         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 707     )
 708     body = bracket_split_build_line(
 709         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 710     )
 711     tail = bracket_split_build_line(
 712         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 713     )
 714     bracket_split_succeeded_or_raise(head, body, tail)
 715     return _RHSResult(head, body, tail, opening_bracket, closing_bracket)
 716
 717
 718 def _maybe_split_omitting_optional_parens(
 719     rhs: _RHSResult,
 720     line: Line,
 721     line_length: int,
 722     features: Collection[Feature] = (),
 723     omit: Collection[LeafID] = (),
 724 ) -> Iterator[Line]:
 725     if (
 726         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 727         # the opening bracket is an optional paren
 728         and rhs.opening_bracket.type == token.LPAR
 729         and not rhs.opening_bracket.value
 730         # the closing bracket is an optional paren
 731         and rhs.closing_bracket.type == token.RPAR
 732         and not rhs.closing_bracket.value
 733         # it's not an import (optional parens are the only thing we can split on
 734         # in this case; attempting a split without them is a waste of time)
 735         and not line.is_import
 736         # there are no standalone comments in the body
 737         and not rhs.body.contains_standalone_comments(0)
 738         # and we can actually remove the parens
 739         and can_omit_invisible_parens(rhs.body, line_length)
 740     ):
 741         omit = {id(rhs.closing_bracket), *omit}
 742         try:
 743             # The _RHSResult Omitting Optional Parens.
 744             rhs_oop = _first_right_hand_split(line, omit=omit)
 745             if not (
 746                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 747                 # the split is right after `=`
 748                 and len(rhs.head.leaves) >= 2
 749                 and rhs.head.leaves[-2].type == token.EQUAL
 750                 # the left side of assignement contains brackets
 751                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 752                 # the left side of assignment is short enough (the -1 is for the ending
 753                 # optional paren)
 754                 and is_line_short_enough(rhs.head, line_length=line_length - 1)
 755                 # the left side of assignment won't explode further because of magic
 756                 # trailing comma
 757                 and rhs.head.magic_trailing_comma is None
 758                 # the split by omitting optional parens isn't preferred by some other
 759                 # reason
 760                 and not _prefer_split_rhs_oop(rhs_oop, line_length=line_length)
 761             ):
 762                 yield from _maybe_split_omitting_optional_parens(
 763                     rhs_oop, line, line_length, features=features, omit=omit
 764                 )
 765                 return
 766
 767         except CannotSplit as e:
 768             if not (
 769                 can_be_split(rhs.body)
 770                 or is_line_short_enough(rhs.body, line_length=line_length)
 771             ):
 772                 raise CannotSplit(
 773                     "Splitting failed, body is still too long and can't be split."
 774                 ) from e
 775
 776             elif (
 777                 rhs.head.contains_multiline_strings()
 778                 or rhs.tail.contains_multiline_strings()
 779             ):
 780                 raise CannotSplit(
 781                     "The current optional pair of parentheses is bound to fail to"
 782                     " satisfy the splitting algorithm because the head or the tail"
 783                     " contains multiline strings which by definition never fit one"
 784                     " line."
 785                 ) from e
 786
 787     ensure_visible(rhs.opening_bracket)
 788     ensure_visible(rhs.closing_bracket)
 789     for result in (rhs.head, rhs.body, rhs.tail):
 790         if result:
 791             yield result
 792
 793
 794 def _prefer_split_rhs_oop(rhs_oop: _RHSResult, line_length: int) -> bool:
 795     """
 796     Returns whether we should prefer the result from a split omitting optional parens.
 797     """
 798     has_closing_bracket_after_assign = False
 799     for leaf in reversed(rhs_oop.head.leaves):
 800         if leaf.type == token.EQUAL:
 801             break
 802         if leaf.type in CLOSING_BRACKETS:
 803             has_closing_bracket_after_assign = True
 804             break
 805     return (
 806         # contains matching brackets after the `=` (done by checking there is a
 807         # closing bracket)
 808         has_closing_bracket_after_assign
 809         or (
 810             # the split is actually from inside the optional parens (done by checking
 811             # the first line still contains the `=`)
 812             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 813             # the first line is short enough
 814             and is_line_short_enough(rhs_oop.head, line_length=line_length)
 815         )
 816         # contains unsplittable type ignore
 817         or rhs_oop.head.contains_unsplittable_type_ignore()
 818         or rhs_oop.body.contains_unsplittable_type_ignore()
 819         or rhs_oop.tail.contains_unsplittable_type_ignore()
 820     )
 821
 822
 823 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 824     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 825
 826     Do nothing otherwise.
 827
 828     A left- or right-hand split is based on a pair of brackets. Content before
 829     (and including) the opening bracket is left on one line, content inside the
 830     brackets is put on a separate line, and finally content starting with and
 831     following the closing bracket is put on a separate line.
 832
 833     Those are called `head`, `body`, and `tail`, respectively. If the split
 834     produced the same line (all content in `head`) or ended up with an empty `body`
 835     and the `tail` is just the closing bracket, then it's considered failed.
 836     """
 837     tail_len = len(str(tail).strip())
 838     if not body:
 839         if tail_len == 0:
 840             raise CannotSplit("Splitting brackets produced the same line")
 841
 842         elif tail_len < 3:
 843             raise CannotSplit(
 844                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 845                 " not worth it"
 846             )
 847
 848
 849 def bracket_split_build_line(
 850     leaves: List[Leaf],
 851     original: Line,
 852     opening_bracket: Leaf,
 853     *,
 854     component: _BracketSplitComponent,
 855 ) -> Line:
 856     """Return a new line with given `leaves` and respective comments from `original`.
 857
 858     If it's the head component, brackets will be tracked so trailing commas are
 859     respected.
 860
 861     If it's the body component, the result line is one-indented inside brackets and as
 862     such has its first leaf's prefix normalized and a trailing comma added when
 863     expected.
 864     """
 865     result = Line(mode=original.mode, depth=original.depth)
 866     if component is _BracketSplitComponent.body:
 867         result.inside_brackets = True
 868         result.depth += 1
 869         if leaves:
 870             # Since body is a new indent level, remove spurious leading whitespace.
 871             normalize_prefix(leaves[0], inside_brackets=True)
 872             # Ensure a trailing comma for imports and standalone function arguments, but
 873             # be careful not to add one after any comments or within type annotations.
 874             no_commas = (
 875                 original.is_def
 876                 and opening_bracket.value == "("
 877                 and not any(leaf.type == token.COMMA for leaf in leaves)
 878                 # In particular, don't add one within a parenthesized return annotation.
 879                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 880                 # be defined directly in the parent node, the parent of the parent ...
 881                 # and so on depending on how complex the return annotation is.
 882                 # This isn't perfect and there's some false negatives but they are in
 883                 # contexts were a comma is actually fine.
 884                 and not any(
 885                     node.prev_sibling.type == RARROW
 886                     for node in (
 887                         leaves[0].parent,
 888                         getattr(leaves[0].parent, "parent", None),
 889                     )
 890                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 891                 )
 892             )
 893
 894             if original.is_import or no_commas:
 895                 for i in range(len(leaves) - 1, -1, -1):
 896                     if leaves[i].type == STANDALONE_COMMENT:
 897                         continue
 898
 899                     if leaves[i].type != token.COMMA:
 900                         new_comma = Leaf(token.COMMA, ",")
 901                         leaves.insert(i + 1, new_comma)
 902                     break
 903
 904     leaves_to_track: Set[LeafID] = set()
 905     if (
 906         Preview.handle_trailing_commas_in_head in original.mode
 907         and component is _BracketSplitComponent.head
 908     ):
 909         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
 910     # Populate the line
 911     for leaf in leaves:
 912         result.append(
 913             leaf,
 914             preformatted=True,
 915             track_bracket=id(leaf) in leaves_to_track,
 916         )
 917         for comment_after in original.comments_after(leaf):
 918             result.append(comment_after, preformatted=True)
 919     if component is _BracketSplitComponent.body and should_split_line(
 920         result, opening_bracket
 921     ):
 922         result.should_split_rhs = True
 923     return result
 924
 925
 926 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 927     """Normalize prefix of the first leaf in every line returned by `split_func`.
 928
 929     This is a decorator over relevant split functions.
 930     """
 931
 932     @wraps(split_func)
 933     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 934         for split_line in split_func(line, features):
 935             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 936             yield split_line
 937
 938     return split_wrapper
 939
 940
 941 @dont_increase_indentation
 942 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 943     """Split according to delimiters of the highest priority.
 944
 945     If the appropriate Features are given, the split will add trailing commas
 946     also in function signatures and calls that contain `*` and `**`.
 947     """
 948     try:
 949         last_leaf = line.leaves[-1]
 950     except IndexError:
 951         raise CannotSplit("Line empty") from None
 952
 953     bt = line.bracket_tracker
 954     try:
 955         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 956     except ValueError:
 957         raise CannotSplit("No delimiters found") from None
 958
 959     if delimiter_priority == DOT_PRIORITY:
 960         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 961             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 962
 963     current_line = Line(
 964         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 965     )
 966     lowest_depth = sys.maxsize
 967     trailing_comma_safe = True
 968
 969     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 970         """Append `leaf` to current line or to new line if appending impossible."""
 971         nonlocal current_line
 972         try:
 973             current_line.append_safe(leaf, preformatted=True)
 974         except ValueError:
 975             yield current_line
 976
 977             current_line = Line(
 978                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 979             )
 980             current_line.append(leaf)
 981
 982     for leaf in line.leaves:
 983         yield from append_to_line(leaf)
 984
 985         for comment_after in line.comments_after(leaf):
 986             yield from append_to_line(comment_after)
 987
 988         lowest_depth = min(lowest_depth, leaf.bracket_depth)
 989         if leaf.bracket_depth == lowest_depth:
 990             if is_vararg(leaf, within={syms.typedargslist}):
 991                 trailing_comma_safe = (
 992                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
 993                 )
 994             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
 995                 trailing_comma_safe = (
 996                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
 997                 )
 998
 999         leaf_priority = bt.delimiters.get(id(leaf))
1000         if leaf_priority == delimiter_priority:
1001             yield current_line
1002
1003             current_line = Line(
1004                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1005             )
1006     if current_line:
1007         if (
1008             trailing_comma_safe
1009             and delimiter_priority == COMMA_PRIORITY
1010             and current_line.leaves[-1].type != token.COMMA
1011             and current_line.leaves[-1].type != STANDALONE_COMMENT
1012         ):
1013             new_comma = Leaf(token.COMMA, ",")
1014             current_line.append(new_comma)
1015         yield current_line
1016
1017
1018 @dont_increase_indentation
1019 def standalone_comment_split(
1020     line: Line, features: Collection[Feature] = ()
1021 ) -> Iterator[Line]:
1022     """Split standalone comments from the rest of the line."""
1023     if not line.contains_standalone_comments(0):
1024         raise CannotSplit("Line does not have any standalone comments")
1025
1026     current_line = Line(
1027         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1028     )
1029
1030     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1031         """Append `leaf` to current line or to new line if appending impossible."""
1032         nonlocal current_line
1033         try:
1034             current_line.append_safe(leaf, preformatted=True)
1035         except ValueError:
1036             yield current_line
1037
1038             current_line = Line(
1039                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1040             )
1041             current_line.append(leaf)
1042
1043     for leaf in line.leaves:
1044         yield from append_to_line(leaf)
1045
1046         for comment_after in line.comments_after(leaf):
1047             yield from append_to_line(comment_after)
1048
1049     if current_line:
1050         yield current_line
1051
1052
1053 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1054     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1055     else.
1056
1057     Note: don't use backslashes for formatting or you'll lose your voting rights.
1058     """
1059     if not inside_brackets:
1060         spl = leaf.prefix.split("#")
1061         if "\\" not in spl[0]:
1062             nl_count = spl[-1].count("\n")
1063             if len(spl) > 1:
1064                 nl_count -= 1
1065             leaf.prefix = "\n" * nl_count
1066             return
1067
1068     leaf.prefix = ""
1069
1070
1071 def normalize_invisible_parens(
1072     node: Node, parens_after: Set[str], *, preview: bool
1073 ) -> None:
1074     """Make existing optional parentheses invisible or create new ones.
1075
1076     `parens_after` is a set of string leaf values immediately after which parens
1077     should be put.
1078
1079     Standardizes on visible parentheses for single-element tuples, and keeps
1080     existing visible parentheses for other tuples and generator expressions.
1081     """
1082     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
1083         if pc.value in FMT_OFF:
1084             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1085             return
1086     check_lpar = False
1087     for index, child in enumerate(list(node.children)):
1088         # Fixes a bug where invisible parens are not properly stripped from
1089         # assignment statements that contain type annotations.
1090         if isinstance(child, Node) and child.type == syms.annassign:
1091             normalize_invisible_parens(
1092                 child, parens_after=parens_after, preview=preview
1093             )
1094
1095         # Add parentheses around long tuple unpacking in assignments.
1096         if (
1097             index == 0
1098             and isinstance(child, Node)
1099             and child.type == syms.testlist_star_expr
1100         ):
1101             check_lpar = True
1102
1103         if check_lpar:
1104             if (
1105                 preview
1106                 and child.type == syms.atom
1107                 and node.type == syms.for_stmt
1108                 and isinstance(child.prev_sibling, Leaf)
1109                 and child.prev_sibling.type == token.NAME
1110                 and child.prev_sibling.value == "for"
1111             ):
1112                 if maybe_make_parens_invisible_in_atom(
1113                     child,
1114                     parent=node,
1115                     remove_brackets_around_comma=True,
1116                 ):
1117                     wrap_in_parentheses(node, child, visible=False)
1118             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
1119                 remove_with_parens(child, node)
1120             elif child.type == syms.atom:
1121                 if maybe_make_parens_invisible_in_atom(
1122                     child,
1123                     parent=node,
1124                 ):
1125                     wrap_in_parentheses(node, child, visible=False)
1126             elif is_one_tuple(child):
1127                 wrap_in_parentheses(node, child, visible=True)
1128             elif node.type == syms.import_from:
1129                 # "import from" nodes store parentheses directly as part of
1130                 # the statement
1131                 if is_lpar_token(child):
1132                     assert is_rpar_token(node.children[-1])
1133                     # make parentheses invisible
1134                     child.value = ""
1135                     node.children[-1].value = ""
1136                 elif child.type != token.STAR:
1137                     # insert invisible parentheses
1138                     node.insert_child(index, Leaf(token.LPAR, ""))
1139                     node.append_child(Leaf(token.RPAR, ""))
1140                 break
1141             elif (
1142                 index == 1
1143                 and child.type == token.STAR
1144                 and node.type == syms.except_clause
1145             ):
1146                 # In except* (PEP 654), the star is actually part of
1147                 # of the keyword. So we need to skip the insertion of
1148                 # invisible parentheses to work more precisely.
1149                 continue
1150
1151             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1152                 wrap_in_parentheses(node, child, visible=False)
1153
1154         comma_check = child.type == token.COMMA if preview else False
1155
1156         check_lpar = isinstance(child, Leaf) and (
1157             child.value in parens_after or comma_check
1158         )
1159
1160
1161 def remove_await_parens(node: Node) -> None:
1162     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1163         if (
1164             node.children[1].type == syms.atom
1165             and node.children[1].children[0].type == token.LPAR
1166         ):
1167             if maybe_make_parens_invisible_in_atom(
1168                 node.children[1],
1169                 parent=node,
1170                 remove_brackets_around_comma=True,
1171             ):
1172                 wrap_in_parentheses(node, node.children[1], visible=False)
1173
1174             # Since await is an expression we shouldn't remove
1175             # brackets in cases where this would change
1176             # the AST due to operator precedence.
1177             # Therefore we only aim to remove brackets around
1178             # power nodes that aren't also await expressions themselves.
1179             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1180             # N.B. We've still removed any redundant nested brackets though :)
1181             opening_bracket = cast(Leaf, node.children[1].children[0])
1182             closing_bracket = cast(Leaf, node.children[1].children[-1])
1183             bracket_contents = cast(Node, node.children[1].children[1])
1184             if bracket_contents.type != syms.power:
1185                 ensure_visible(opening_bracket)
1186                 ensure_visible(closing_bracket)
1187             elif (
1188                 bracket_contents.type == syms.power
1189                 and bracket_contents.children[0].type == token.AWAIT
1190             ):
1191                 ensure_visible(opening_bracket)
1192                 ensure_visible(closing_bracket)
1193                 # If we are in a nested await then recurse down.
1194                 remove_await_parens(bracket_contents)
1195
1196
1197 def remove_with_parens(node: Node, parent: Node) -> None:
1198     """Recursively hide optional parens in `with` statements."""
1199     # Removing all unnecessary parentheses in with statements in one pass is a tad
1200     # complex as different variations of bracketed statements result in pretty
1201     # different parse trees:
1202     #
1203     # with (open("file")) as f:                       # this is an asexpr_test
1204     #     ...
1205     #
1206     # with (open("file") as f):                       # this is an atom containing an
1207     #     ...                                         # asexpr_test
1208     #
1209     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1210     #     ...                                         # asexpr_test
1211     #
1212     # with (open("file") as f, open("file") as f):    # an atom containing a
1213     #     ...                                         # testlist_gexp which then
1214     #                                                 # contains multiple asexpr_test(s)
1215     if node.type == syms.atom:
1216         if maybe_make_parens_invisible_in_atom(
1217             node,
1218             parent=parent,
1219             remove_brackets_around_comma=True,
1220         ):
1221             wrap_in_parentheses(parent, node, visible=False)
1222         if isinstance(node.children[1], Node):
1223             remove_with_parens(node.children[1], node)
1224     elif node.type == syms.testlist_gexp:
1225         for child in node.children:
1226             if isinstance(child, Node):
1227                 remove_with_parens(child, node)
1228     elif node.type == syms.asexpr_test and not any(
1229         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1230     ):
1231         if maybe_make_parens_invisible_in_atom(
1232             node.children[0],
1233             parent=node,
1234             remove_brackets_around_comma=True,
1235         ):
1236             wrap_in_parentheses(node, node.children[0], visible=False)
1237
1238
1239 def maybe_make_parens_invisible_in_atom(
1240     node: LN,
1241     parent: LN,
1242     remove_brackets_around_comma: bool = False,
1243 ) -> bool:
1244     """If it's safe, make the parens in the atom `node` invisible, recursively.
1245     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1246     as they are redundant.
1247
1248     Returns whether the node should itself be wrapped in invisible parentheses.
1249     """
1250     if (
1251         node.type != syms.atom
1252         or is_empty_tuple(node)
1253         or is_one_tuple(node)
1254         or (is_yield(node) and parent.type != syms.expr_stmt)
1255         or (
1256             # This condition tries to prevent removing non-optional brackets
1257             # around a tuple, however, can be a bit overzealous so we provide
1258             # and option to skip this check for `for` and `with` statements.
1259             not remove_brackets_around_comma
1260             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1261         )
1262     ):
1263         return False
1264
1265     if is_walrus_assignment(node):
1266         if parent.type in [
1267             syms.annassign,
1268             syms.expr_stmt,
1269             syms.assert_stmt,
1270             syms.return_stmt,
1271             syms.except_clause,
1272             syms.funcdef,
1273             # these ones aren't useful to end users, but they do please fuzzers
1274             syms.for_stmt,
1275             syms.del_stmt,
1276         ]:
1277             return False
1278
1279     first = node.children[0]
1280     last = node.children[-1]
1281     if is_lpar_token(first) and is_rpar_token(last):
1282         middle = node.children[1]
1283         # make parentheses invisible
1284         first.value = ""
1285         last.value = ""
1286         maybe_make_parens_invisible_in_atom(
1287             middle,
1288             parent=parent,
1289             remove_brackets_around_comma=remove_brackets_around_comma,
1290         )
1291
1292         if is_atom_with_invisible_parens(middle):
1293             # Strip the invisible parens from `middle` by replacing
1294             # it with the child in-between the invisible parens
1295             middle.replace(middle.children[1])
1296
1297         return False
1298
1299     return True
1300
1301
1302 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1303     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1304
1305     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1306         return False
1307
1308     # We're essentially checking if the body is delimited by commas and there's more
1309     # than one of them (we're excluding the trailing comma and if the delimiter priority
1310     # is still commas, that means there's more).
1311     exclude = set()
1312     trailing_comma = False
1313     try:
1314         last_leaf = line.leaves[-1]
1315         if last_leaf.type == token.COMMA:
1316             trailing_comma = True
1317             exclude.add(id(last_leaf))
1318         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1319     except (IndexError, ValueError):
1320         return False
1321
1322     return max_priority == COMMA_PRIORITY and (
1323         (line.mode.magic_trailing_comma and trailing_comma)
1324         # always explode imports
1325         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1326     )
1327
1328
1329 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1330     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1331
1332     Brackets can be omitted if the entire trailer up to and including
1333     a preceding closing bracket fits in one line.
1334
1335     Yielded sets are cumulative (contain results of previous yields, too).  First
1336     set is empty, unless the line should explode, in which case bracket pairs until
1337     the one that needs to explode are omitted.
1338     """
1339
1340     omit: Set[LeafID] = set()
1341     if not line.magic_trailing_comma:
1342         yield omit
1343
1344     length = 4 * line.depth
1345     opening_bracket: Optional[Leaf] = None
1346     closing_bracket: Optional[Leaf] = None
1347     inner_brackets: Set[LeafID] = set()
1348     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1349         length += leaf_length
1350         if length > line_length:
1351             break
1352
1353         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1354         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1355             break
1356
1357         if opening_bracket:
1358             if leaf is opening_bracket:
1359                 opening_bracket = None
1360             elif leaf.type in CLOSING_BRACKETS:
1361                 prev = line.leaves[index - 1] if index > 0 else None
1362                 if (
1363                     prev
1364                     and prev.type == token.COMMA
1365                     and leaf.opening_bracket is not None
1366                     and not is_one_sequence_between(
1367                         leaf.opening_bracket, leaf, line.leaves
1368                     )
1369                 ):
1370                     # Never omit bracket pairs with trailing commas.
1371                     # We need to explode on those.
1372                     break
1373
1374                 inner_brackets.add(id(leaf))
1375         elif leaf.type in CLOSING_BRACKETS:
1376             prev = line.leaves[index - 1] if index > 0 else None
1377             if prev and prev.type in OPENING_BRACKETS:
1378                 # Empty brackets would fail a split so treat them as "inner"
1379                 # brackets (e.g. only add them to the `omit` set if another
1380                 # pair of brackets was good enough.
1381                 inner_brackets.add(id(leaf))
1382                 continue
1383
1384             if closing_bracket:
1385                 omit.add(id(closing_bracket))
1386                 omit.update(inner_brackets)
1387                 inner_brackets.clear()
1388                 yield omit
1389
1390             if (
1391                 prev
1392                 and prev.type == token.COMMA
1393                 and leaf.opening_bracket is not None
1394                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1395             ):
1396                 # Never omit bracket pairs with trailing commas.
1397                 # We need to explode on those.
1398                 break
1399
1400             if leaf.value:
1401                 opening_bracket = leaf.opening_bracket
1402                 closing_bracket = leaf
1403
1404
1405 def run_transformer(
1406     line: Line,
1407     transform: Transformer,
1408     mode: Mode,
1409     features: Collection[Feature],
1410     *,
1411     line_str: str = "",
1412 ) -> List[Line]:
1413     if not line_str:
1414         line_str = line_to_string(line)
1415     result: List[Line] = []
1416     for transformed_line in transform(line, features):
1417         if str(transformed_line).strip("\n") == line_str:
1418             raise CannotTransform("Line transformer returned an unchanged result")
1419
1420         result.extend(transform_line(transformed_line, mode=mode, features=features))
1421
1422     features_set = set(features)
1423     if (
1424         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1425         or transform.__class__.__name__ != "rhs"
1426         or not line.bracket_tracker.invisible
1427         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1428         or line.contains_multiline_strings()
1429         or result[0].contains_uncollapsable_type_comments()
1430         or result[0].contains_unsplittable_type_ignore()
1431         or is_line_short_enough(result[0], line_length=mode.line_length)
1432         # If any leaves have no parents (which _can_ occur since
1433         # `transform(line)` potentially destroys the line's underlying node
1434         # structure), then we can't proceed. Doing so would cause the below
1435         # call to `append_leaves()` to fail.
1436         or any(leaf.parent is None for leaf in line.leaves)
1437     ):
1438         return result
1439
1440     line_copy = line.clone()
1441     append_leaves(line_copy, line, line.leaves)
1442     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1443     second_opinion = run_transformer(
1444         line_copy, transform, mode, features_fop, line_str=line_str
1445     )
1446     if all(
1447         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1448     ):
1449         result = second_opinion
1450     return result