.vim/bundle/black/src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4
   5 import sys
   6 from dataclasses import replace
   7 from enum import Enum, auto
   8 from functools import partial, wraps
   9 from typing import Collection, Iterator, List, Optional, Set, Union, cast
  10
  11 from black.brackets import (
  12     COMMA_PRIORITY,
  13     DOT_PRIORITY,
  14     get_leaves_inside_matching_brackets,
  15     max_delimiter_priority_in_atom,
  16 )
  17 from black.comments import FMT_OFF, generate_comments, list_comments
  18 from black.lines import (
  19     Line,
  20     RHSResult,
  21     append_leaves,
  22     can_be_split,
  23     can_omit_invisible_parens,
  24     is_line_short_enough,
  25     line_to_string,
  26 )
  27 from black.mode import Feature, Mode, Preview
  28 from black.nodes import (
  29     ASSIGNMENTS,
  30     BRACKETS,
  31     CLOSING_BRACKETS,
  32     OPENING_BRACKETS,
  33     RARROW,
  34     STANDALONE_COMMENT,
  35     STATEMENT,
  36     WHITESPACE,
  37     Visitor,
  38     ensure_visible,
  39     is_arith_like,
  40     is_async_stmt_or_funcdef,
  41     is_atom_with_invisible_parens,
  42     is_docstring,
  43     is_empty_tuple,
  44     is_lpar_token,
  45     is_multiline_string,
  46     is_name_token,
  47     is_one_sequence_between,
  48     is_one_tuple,
  49     is_rpar_token,
  50     is_stub_body,
  51     is_stub_suite,
  52     is_tuple_containing_walrus,
  53     is_type_ignore_comment_string,
  54     is_vararg,
  55     is_walrus_assignment,
  56     is_yield,
  57     syms,
  58     wrap_in_parentheses,
  59 )
  60 from black.numerics import normalize_numeric_literal
  61 from black.strings import (
  62     fix_docstring,
  63     get_string_prefix,
  64     normalize_string_prefix,
  65     normalize_string_quotes,
  66     normalize_unicode_escape_sequences,
  67 )
  68 from black.trans import (
  69     CannotTransform,
  70     StringMerger,
  71     StringParenStripper,
  72     StringParenWrapper,
  73     StringSplitter,
  74     Transformer,
  75     hug_power_op,
  76 )
  77 from blib2to3.pgen2 import token
  78 from blib2to3.pytree import Leaf, Node
  79
  80 # types
  81 LeafID = int
  82 LN = Union[Leaf, Node]
  83
  84
  85 class CannotSplit(CannotTransform):
  86     """A readable split that fits the allotted line length is impossible."""
  87
  88
  89 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  90 # See also https://github.com/mypyc/mypyc/issues/827.
  91 class LineGenerator(Visitor[Line]):
  92     """Generates reformatted Line objects.  Empty lines are not emitted.
  93
  94     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  95     in ways that will no longer stringify to valid Python code on the tree.
  96     """
  97
  98     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  99         self.mode = mode
 100         self.features = features
 101         self.current_line: Line
 102         self.__post_init__()
 103
 104     def line(self, indent: int = 0) -> Iterator[Line]:
 105         """Generate a line.
 106
 107         If the line is empty, only emit if it makes sense.
 108         If the line is too long, split it first and then generate.
 109
 110         If any lines were generated, set up a new current_line.
 111         """
 112         if not self.current_line:
 113             self.current_line.depth += indent
 114             return  # Line is empty, don't emit. Creating a new one unnecessary.
 115
 116         if (
 117             Preview.improved_async_statements_handling in self.mode
 118             and len(self.current_line.leaves) == 1
 119             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
 120         ):
 121             # Special case for async def/for/with statements. `visit_async_stmt`
 122             # adds an `ASYNC` leaf then visits the child def/for/with statement
 123             # nodes. Line yields from those nodes shouldn't treat the former
 124             # `ASYNC` leaf as a complete line.
 125             return
 126
 127         complete_line = self.current_line
 128         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 129         yield complete_line
 130
 131     def visit_default(self, node: LN) -> Iterator[Line]:
 132         """Default `visit_*()` implementation. Recurses to children of `node`."""
 133         if isinstance(node, Leaf):
 134             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 135             for comment in generate_comments(node):
 136                 if any_open_brackets:
 137                     # any comment within brackets is subject to splitting
 138                     self.current_line.append(comment)
 139                 elif comment.type == token.COMMENT:
 140                     # regular trailing comment
 141                     self.current_line.append(comment)
 142                     yield from self.line()
 143
 144                 else:
 145                     # regular standalone comment
 146                     yield from self.line()
 147
 148                     self.current_line.append(comment)
 149                     yield from self.line()
 150
 151             normalize_prefix(node, inside_brackets=any_open_brackets)
 152             if self.mode.string_normalization and node.type == token.STRING:
 153                 node.value = normalize_string_prefix(node.value)
 154                 node.value = normalize_string_quotes(node.value)
 155             if node.type == token.NUMBER:
 156                 normalize_numeric_literal(node)
 157             if node.type not in WHITESPACE:
 158                 self.current_line.append(node)
 159         yield from super().visit_default(node)
 160
 161     def visit_test(self, node: Node) -> Iterator[Line]:
 162         """Visit an `x if y else z` test"""
 163
 164         if Preview.parenthesize_conditional_expressions in self.mode:
 165             already_parenthesized = (
 166                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 167             )
 168
 169             if not already_parenthesized:
 170                 lpar = Leaf(token.LPAR, "")
 171                 rpar = Leaf(token.RPAR, "")
 172                 node.insert_child(0, lpar)
 173                 node.append_child(rpar)
 174
 175         yield from self.visit_default(node)
 176
 177     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 178         """Increase indentation level, maybe yield a line."""
 179         # In blib2to3 INDENT never holds comments.
 180         yield from self.line(+1)
 181         yield from self.visit_default(node)
 182
 183     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 184         """Decrease indentation level, maybe yield a line."""
 185         # The current line might still wait for trailing comments.  At DEDENT time
 186         # there won't be any (they would be prefixes on the preceding NEWLINE).
 187         # Emit the line then.
 188         yield from self.line()
 189
 190         # While DEDENT has no value, its prefix may contain standalone comments
 191         # that belong to the current indentation level.  Get 'em.
 192         yield from self.visit_default(node)
 193
 194         # Finally, emit the dedent.
 195         yield from self.line(-1)
 196
 197     def visit_stmt(
 198         self, node: Node, keywords: Set[str], parens: Set[str]
 199     ) -> Iterator[Line]:
 200         """Visit a statement.
 201
 202         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 203         `def`, `with`, `class`, `assert`, and assignments.
 204
 205         The relevant Python language `keywords` for a given statement will be
 206         NAME leaves within it. This methods puts those on a separate line.
 207
 208         `parens` holds a set of string leaf values immediately after which
 209         invisible parens should be put.
 210         """
 211         normalize_invisible_parens(
 212             node, parens_after=parens, mode=self.mode, features=self.features
 213         )
 214         for child in node.children:
 215             if is_name_token(child) and child.value in keywords:
 216                 yield from self.line()
 217
 218             yield from self.visit(child)
 219
 220     def visit_typeparams(self, node: Node) -> Iterator[Line]:
 221         yield from self.visit_default(node)
 222         node.children[0].prefix = ""
 223
 224     def visit_typevartuple(self, node: Node) -> Iterator[Line]:
 225         yield from self.visit_default(node)
 226         node.children[1].prefix = ""
 227
 228     def visit_paramspec(self, node: Node) -> Iterator[Line]:
 229         yield from self.visit_default(node)
 230         node.children[1].prefix = ""
 231
 232     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 233         if Preview.wrap_long_dict_values_in_parens in self.mode:
 234             for i, child in enumerate(node.children):
 235                 if i == 0:
 236                     continue
 237                 if node.children[i - 1].type == token.COLON:
 238                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 239                         if maybe_make_parens_invisible_in_atom(
 240                             child,
 241                             parent=node,
 242                             remove_brackets_around_comma=False,
 243                         ):
 244                             wrap_in_parentheses(node, child, visible=False)
 245                     else:
 246                         wrap_in_parentheses(node, child, visible=False)
 247         yield from self.visit_default(node)
 248
 249     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 250         """Visit function definition."""
 251         yield from self.line()
 252
 253         # Remove redundant brackets around return type annotation.
 254         is_return_annotation = False
 255         for child in node.children:
 256             if child.type == token.RARROW:
 257                 is_return_annotation = True
 258             elif is_return_annotation:
 259                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 260                     if maybe_make_parens_invisible_in_atom(
 261                         child,
 262                         parent=node,
 263                         remove_brackets_around_comma=False,
 264                     ):
 265                         wrap_in_parentheses(node, child, visible=False)
 266                 else:
 267                     wrap_in_parentheses(node, child, visible=False)
 268                 is_return_annotation = False
 269
 270         for child in node.children:
 271             yield from self.visit(child)
 272
 273     def visit_match_case(self, node: Node) -> Iterator[Line]:
 274         """Visit either a match or case statement."""
 275         normalize_invisible_parens(
 276             node, parens_after=set(), mode=self.mode, features=self.features
 277         )
 278
 279         yield from self.line()
 280         for child in node.children:
 281             yield from self.visit(child)
 282
 283     def visit_suite(self, node: Node) -> Iterator[Line]:
 284         """Visit a suite."""
 285         if (
 286             self.mode.is_pyi or Preview.dummy_implementations in self.mode
 287         ) and is_stub_suite(node):
 288             yield from self.visit(node.children[2])
 289         else:
 290             yield from self.visit_default(node)
 291
 292     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 293         """Visit a statement without nested statements."""
 294         prev_type: Optional[int] = None
 295         for child in node.children:
 296             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 297                 wrap_in_parentheses(node, child, visible=False)
 298             prev_type = child.type
 299
 300         is_suite_like = node.parent and node.parent.type in STATEMENT
 301         if is_suite_like:
 302             if (
 303                 self.mode.is_pyi or Preview.dummy_implementations in self.mode
 304             ) and is_stub_body(node):
 305                 yield from self.visit_default(node)
 306             else:
 307                 yield from self.line(+1)
 308                 yield from self.visit_default(node)
 309                 yield from self.line(-1)
 310
 311         else:
 312             if (
 313                 not (self.mode.is_pyi or Preview.dummy_implementations in self.mode)
 314                 or not node.parent
 315                 or not is_stub_suite(node.parent)
 316             ):
 317                 yield from self.line()
 318             yield from self.visit_default(node)
 319
 320     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 321         """Visit `async def`, `async for`, `async with`."""
 322         yield from self.line()
 323
 324         children = iter(node.children)
 325         for child in children:
 326             yield from self.visit(child)
 327
 328             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 329                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 330                 # line.
 331                 break
 332
 333         internal_stmt = next(children)
 334         if Preview.improved_async_statements_handling in self.mode:
 335             yield from self.visit(internal_stmt)
 336         else:
 337             for child in internal_stmt.children:
 338                 yield from self.visit(child)
 339
 340     def visit_decorators(self, node: Node) -> Iterator[Line]:
 341         """Visit decorators."""
 342         for child in node.children:
 343             yield from self.line()
 344             yield from self.visit(child)
 345
 346     def visit_power(self, node: Node) -> Iterator[Line]:
 347         for idx, leaf in enumerate(node.children[:-1]):
 348             next_leaf = node.children[idx + 1]
 349
 350             if not isinstance(leaf, Leaf):
 351                 continue
 352
 353             value = leaf.value.lower()
 354             if (
 355                 leaf.type == token.NUMBER
 356                 and next_leaf.type == syms.trailer
 357                 # Ensure that we are in an attribute trailer
 358                 and next_leaf.children[0].type == token.DOT
 359                 # It shouldn't wrap hexadecimal, binary and octal literals
 360                 and not value.startswith(("0x", "0b", "0o"))
 361                 # It shouldn't wrap complex literals
 362                 and "j" not in value
 363             ):
 364                 wrap_in_parentheses(node, leaf)
 365
 366         remove_await_parens(node)
 367
 368         yield from self.visit_default(node)
 369
 370     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 371         """Remove a semicolon and put the other statement on a separate line."""
 372         yield from self.line()
 373
 374     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 375         """End of file. Process outstanding comments and end with a newline."""
 376         yield from self.visit_default(leaf)
 377         yield from self.line()
 378
 379     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 380         if not self.current_line.bracket_tracker.any_open_brackets():
 381             yield from self.line()
 382         yield from self.visit_default(leaf)
 383
 384     def visit_factor(self, node: Node) -> Iterator[Line]:
 385         """Force parentheses between a unary op and a binary power:
 386
 387         -2 ** 8 -> -(2 ** 8)
 388         """
 389         _operator, operand = node.children
 390         if (
 391             operand.type == syms.power
 392             and len(operand.children) == 3
 393             and operand.children[1].type == token.DOUBLESTAR
 394         ):
 395             lpar = Leaf(token.LPAR, "(")
 396             rpar = Leaf(token.RPAR, ")")
 397             index = operand.remove() or 0
 398             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 399         yield from self.visit_default(node)
 400
 401     def visit_tname(self, node: Node) -> Iterator[Line]:
 402         """
 403         Add potential parentheses around types in function parameter lists to be made
 404         into real parentheses in case the type hint is too long to fit on a line
 405         Examples:
 406         def foo(a: int, b: float = 7): ...
 407
 408         ->
 409
 410         def foo(a: (int), b: (float) = 7): ...
 411         """
 412         if Preview.parenthesize_long_type_hints in self.mode:
 413             assert len(node.children) == 3
 414             if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):
 415                 wrap_in_parentheses(node, node.children[2], visible=False)
 416
 417         yield from self.visit_default(node)
 418
 419     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 420         if Preview.hex_codes_in_unicode_sequences in self.mode:
 421             normalize_unicode_escape_sequences(leaf)
 422
 423         if is_docstring(leaf) and "\\\n" not in leaf.value:
 424             # We're ignoring docstrings with backslash newline escapes because changing
 425             # indentation of those changes the AST representation of the code.
 426             if self.mode.string_normalization:
 427                 docstring = normalize_string_prefix(leaf.value)
 428                 # visit_default() does handle string normalization for us, but
 429                 # since this method acts differently depending on quote style (ex.
 430                 # see padding logic below), there's a possibility for unstable
 431                 # formatting as visit_default() is called *after*. To avoid a
 432                 # situation where this function formats a docstring differently on
 433                 # the second pass, normalize it early.
 434                 docstring = normalize_string_quotes(docstring)
 435             else:
 436                 docstring = leaf.value
 437             prefix = get_string_prefix(docstring)
 438             docstring = docstring[len(prefix) :]  # Remove the prefix
 439             quote_char = docstring[0]
 440             # A natural way to remove the outer quotes is to do:
 441             #   docstring = docstring.strip(quote_char)
 442             # but that breaks on """""x""" (which is '""x').
 443             # So we actually need to remove the first character and the next two
 444             # characters but only if they are the same as the first.
 445             quote_len = 1 if docstring[1] != quote_char else 3
 446             docstring = docstring[quote_len:-quote_len]
 447             docstring_started_empty = not docstring
 448             indent = " " * 4 * self.current_line.depth
 449
 450             if is_multiline_string(leaf):
 451                 docstring = fix_docstring(docstring, indent)
 452             else:
 453                 docstring = docstring.strip()
 454
 455             has_trailing_backslash = False
 456             if docstring:
 457                 # Add some padding if the docstring starts / ends with a quote mark.
 458                 if docstring[0] == quote_char:
 459                     docstring = " " + docstring
 460                 if docstring[-1] == quote_char:
 461                     docstring += " "
 462                 if docstring[-1] == "\\":
 463                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 464                     if backslash_count % 2:
 465                         # Odd number of tailing backslashes, add some padding to
 466                         # avoid escaping the closing string quote.
 467                         docstring += " "
 468                         has_trailing_backslash = True
 469             elif not docstring_started_empty:
 470                 docstring = " "
 471
 472             # We could enforce triple quotes at this point.
 473             quote = quote_char * quote_len
 474
 475             # It's invalid to put closing single-character quotes on a new line.
 476             if self.mode and quote_len == 3:
 477                 # We need to find the length of the last line of the docstring
 478                 # to find if we can add the closing quotes to the line without
 479                 # exceeding the maximum line length.
 480                 # If docstring is one line, we don't put the closing quotes on a
 481                 # separate line because it looks ugly (#3320).
 482                 lines = docstring.splitlines()
 483                 last_line_length = len(lines[-1]) if docstring else 0
 484
 485                 # If adding closing quotes would cause the last line to exceed
 486                 # the maximum line length then put a line break before the
 487                 # closing quotes
 488                 if (
 489                     len(lines) > 1
 490                     and last_line_length + quote_len > self.mode.line_length
 491                     and len(indent) + quote_len <= self.mode.line_length
 492                     and not has_trailing_backslash
 493                 ):
 494                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 495                 else:
 496                     leaf.value = prefix + quote + docstring + quote
 497             else:
 498                 leaf.value = prefix + quote + docstring + quote
 499
 500         yield from self.visit_default(leaf)
 501
 502     def __post_init__(self) -> None:
 503         """You are in a twisty little maze of passages."""
 504         self.current_line = Line(mode=self.mode)
 505
 506         v = self.visit_stmt
 507         Ø: Set[str] = set()
 508         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 509         self.visit_if_stmt = partial(
 510             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 511         )
 512         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 513         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 514         self.visit_try_stmt = partial(
 515             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 516         )
 517         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 518         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 519         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 520
 521         # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py
 522         if Preview.parenthesize_long_type_hints in self.mode:
 523             assignments = ASSIGNMENTS | {":"}
 524         else:
 525             assignments = ASSIGNMENTS
 526         self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments)
 527
 528         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 529         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 530         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 531         self.visit_async_funcdef = self.visit_async_stmt
 532         self.visit_decorated = self.visit_decorators
 533
 534         # PEP 634
 535         self.visit_match_stmt = self.visit_match_case
 536         self.visit_case_block = self.visit_match_case
 537
 538
 539 def _hugging_power_ops_line_to_string(
 540     line: Line,
 541     features: Collection[Feature],
 542     mode: Mode,
 543 ) -> Optional[str]:
 544     try:
 545         return line_to_string(next(hug_power_op(line, features, mode)))
 546     except CannotTransform:
 547         return None
 548
 549
 550 def transform_line(
 551     line: Line, mode: Mode, features: Collection[Feature] = ()
 552 ) -> Iterator[Line]:
 553     """Transform a `line`, potentially splitting it into many lines.
 554
 555     They should fit in the allotted `line_length` but might not be able to.
 556
 557     `features` are syntactical features that may be used in the output.
 558     """
 559     if line.is_comment:
 560         yield line
 561         return
 562
 563     line_str = line_to_string(line)
 564
 565     # We need the line string when power operators are hugging to determine if we should
 566     # split the line. Default to line_str, if no power operator are present on the line.
 567     line_str_hugging_power_ops = (
 568         (_hugging_power_ops_line_to_string(line, features, mode) or line_str)
 569         if Preview.fix_power_op_line_length in mode
 570         else line_str
 571     )
 572
 573     ll = mode.line_length
 574     sn = mode.string_normalization
 575     string_merge = StringMerger(ll, sn)
 576     string_paren_strip = StringParenStripper(ll, sn)
 577     string_split = StringSplitter(ll, sn)
 578     string_paren_wrap = StringParenWrapper(ll, sn)
 579
 580     transformers: List[Transformer]
 581     if (
 582         not line.contains_uncollapsable_type_comments()
 583         and not line.should_split_rhs
 584         and not line.magic_trailing_comma
 585         and (
 586             is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
 587             or line.contains_unsplittable_type_ignore()
 588         )
 589         and not (line.inside_brackets and line.contains_standalone_comments())
 590         and not line.contains_implicit_multiline_string_with_comments()
 591     ):
 592         # Only apply basic string preprocessing, since lines shouldn't be split here.
 593         if Preview.string_processing in mode:
 594             transformers = [string_merge, string_paren_strip]
 595         else:
 596             transformers = []
 597     elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
 598         transformers = [left_hand_split]
 599     else:
 600
 601         def _rhs(
 602             self: object, line: Line, features: Collection[Feature], mode: Mode
 603         ) -> Iterator[Line]:
 604             """Wraps calls to `right_hand_split`.
 605
 606             The calls increasingly `omit` right-hand trailers (bracket pairs with
 607             content), meaning the trailers get glued together to split on another
 608             bracket pair instead.
 609             """
 610             for omit in generate_trailers_to_omit(line, mode.line_length):
 611                 lines = list(right_hand_split(line, mode, features, omit=omit))
 612                 # Note: this check is only able to figure out if the first line of the
 613                 # *current* transformation fits in the line length.  This is true only
 614                 # for simple cases.  All others require running more transforms via
 615                 # `transform_line()`.  This check doesn't know if those would succeed.
 616                 if is_line_short_enough(lines[0], mode=mode):
 617                     yield from lines
 618                     return
 619
 620             # All splits failed, best effort split with no omits.
 621             # This mostly happens to multiline strings that are by definition
 622             # reported as not fitting a single line, as well as lines that contain
 623             # trailing commas (those have to be exploded).
 624             yield from right_hand_split(line, mode, features=features)
 625
 626         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 627         # __name__ attribute which is needed in `run_transformer` further down.
 628         # Unfortunately a nested class breaks mypyc too. So a class must be created
 629         # via type ... https://github.com/mypyc/mypyc/issues/884
 630         rhs = type("rhs", (), {"__call__": _rhs})()
 631
 632         if Preview.string_processing in mode:
 633             if line.inside_brackets:
 634                 transformers = [
 635                     string_merge,
 636                     string_paren_strip,
 637                     string_split,
 638                     delimiter_split,
 639                     standalone_comment_split,
 640                     string_paren_wrap,
 641                     rhs,
 642                 ]
 643             else:
 644                 transformers = [
 645                     string_merge,
 646                     string_paren_strip,
 647                     string_split,
 648                     string_paren_wrap,
 649                     rhs,
 650                 ]
 651         else:
 652             if line.inside_brackets:
 653                 transformers = [delimiter_split, standalone_comment_split, rhs]
 654             else:
 655                 transformers = [rhs]
 656     # It's always safe to attempt hugging of power operations and pretty much every line
 657     # could match.
 658     transformers.append(hug_power_op)
 659
 660     for transform in transformers:
 661         # We are accumulating lines in `result` because we might want to abort
 662         # mission and return the original line in the end, or attempt a different
 663         # split altogether.
 664         try:
 665             result = run_transformer(line, transform, mode, features, line_str=line_str)
 666         except CannotTransform:
 667             continue
 668         else:
 669             yield from result
 670             break
 671
 672     else:
 673         yield line
 674
 675
 676 def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
 677     """If a funcdef has a magic trailing comma in the return type, then we should first
 678     split the line with rhs to respect the comma.
 679     """
 680     if Preview.respect_magic_trailing_comma_in_return_type not in mode:
 681         return False
 682
 683     return_type_leaves: List[Leaf] = []
 684     in_return_type = False
 685
 686     for leaf in line.leaves:
 687         if leaf.type == token.COLON:
 688             in_return_type = False
 689         if in_return_type:
 690             return_type_leaves.append(leaf)
 691         if leaf.type == token.RARROW:
 692             in_return_type = True
 693
 694     # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
 695     # couple lines from it.
 696     result = Line(mode=line.mode, depth=line.depth)
 697     leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
 698     for leaf in return_type_leaves:
 699         result.append(
 700             leaf,
 701             preformatted=True,
 702             track_bracket=id(leaf) in leaves_to_track,
 703         )
 704
 705     # we could also return true if the line is too long, and the return type is longer
 706     # than the param list. Or if `should_split_rhs` returns True.
 707     return result.magic_trailing_comma is not None
 708
 709
 710 class _BracketSplitComponent(Enum):
 711     head = auto()
 712     body = auto()
 713     tail = auto()
 714
 715
 716 def left_hand_split(
 717     line: Line, _features: Collection[Feature], mode: Mode
 718 ) -> Iterator[Line]:
 719     """Split line into many lines, starting with the first matching bracket pair.
 720
 721     Note: this usually looks weird, only use this for function definitions.
 722     Prefer RHS otherwise.  This is why this function is not symmetrical with
 723     :func:`right_hand_split` which also handles optional parentheses.
 724     """
 725     tail_leaves: List[Leaf] = []
 726     body_leaves: List[Leaf] = []
 727     head_leaves: List[Leaf] = []
 728     current_leaves = head_leaves
 729     matching_bracket: Optional[Leaf] = None
 730     for leaf in line.leaves:
 731         if (
 732             current_leaves is body_leaves
 733             and leaf.type in CLOSING_BRACKETS
 734             and leaf.opening_bracket is matching_bracket
 735             and isinstance(matching_bracket, Leaf)
 736         ):
 737             ensure_visible(leaf)
 738             ensure_visible(matching_bracket)
 739             current_leaves = tail_leaves if body_leaves else head_leaves
 740         current_leaves.append(leaf)
 741         if current_leaves is head_leaves:
 742             if leaf.type in OPENING_BRACKETS:
 743                 matching_bracket = leaf
 744                 current_leaves = body_leaves
 745     if not matching_bracket:
 746         raise CannotSplit("No brackets found")
 747
 748     head = bracket_split_build_line(
 749         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 750     )
 751     body = bracket_split_build_line(
 752         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 753     )
 754     tail = bracket_split_build_line(
 755         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 756     )
 757     bracket_split_succeeded_or_raise(head, body, tail)
 758     for result in (head, body, tail):
 759         if result:
 760             yield result
 761
 762
 763 def right_hand_split(
 764     line: Line,
 765     mode: Mode,
 766     features: Collection[Feature] = (),
 767     omit: Collection[LeafID] = (),
 768 ) -> Iterator[Line]:
 769     """Split line into many lines, starting with the last matching bracket pair.
 770
 771     If the split was by optional parentheses, attempt splitting without them, too.
 772     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 773     this split.
 774
 775     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 776     """
 777     rhs_result = _first_right_hand_split(line, omit=omit)
 778     yield from _maybe_split_omitting_optional_parens(
 779         rhs_result, line, mode, features=features, omit=omit
 780     )
 781
 782
 783 def _first_right_hand_split(
 784     line: Line,
 785     omit: Collection[LeafID] = (),
 786 ) -> RHSResult:
 787     """Split the line into head, body, tail starting with the last bracket pair.
 788
 789     Note: this function should not have side effects. It's relied upon by
 790     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 791     splitting on the right side of an assignment statement.
 792     """
 793     tail_leaves: List[Leaf] = []
 794     body_leaves: List[Leaf] = []
 795     head_leaves: List[Leaf] = []
 796     current_leaves = tail_leaves
 797     opening_bracket: Optional[Leaf] = None
 798     closing_bracket: Optional[Leaf] = None
 799     for leaf in reversed(line.leaves):
 800         if current_leaves is body_leaves:
 801             if leaf is opening_bracket:
 802                 current_leaves = head_leaves if body_leaves else tail_leaves
 803         current_leaves.append(leaf)
 804         if current_leaves is tail_leaves:
 805             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 806                 opening_bracket = leaf.opening_bracket
 807                 closing_bracket = leaf
 808                 current_leaves = body_leaves
 809     if not (opening_bracket and closing_bracket and head_leaves):
 810         # If there is no opening or closing_bracket that means the split failed and
 811         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 812         # the matching `opening_bracket` wasn't available on `line` anymore.
 813         raise CannotSplit("No brackets found")
 814
 815     tail_leaves.reverse()
 816     body_leaves.reverse()
 817     head_leaves.reverse()
 818     head = bracket_split_build_line(
 819         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 820     )
 821     body = bracket_split_build_line(
 822         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 823     )
 824     tail = bracket_split_build_line(
 825         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 826     )
 827     bracket_split_succeeded_or_raise(head, body, tail)
 828     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 829
 830
 831 def _maybe_split_omitting_optional_parens(
 832     rhs: RHSResult,
 833     line: Line,
 834     mode: Mode,
 835     features: Collection[Feature] = (),
 836     omit: Collection[LeafID] = (),
 837 ) -> Iterator[Line]:
 838     if (
 839         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 840         # the opening bracket is an optional paren
 841         and rhs.opening_bracket.type == token.LPAR
 842         and not rhs.opening_bracket.value
 843         # the closing bracket is an optional paren
 844         and rhs.closing_bracket.type == token.RPAR
 845         and not rhs.closing_bracket.value
 846         # it's not an import (optional parens are the only thing we can split on
 847         # in this case; attempting a split without them is a waste of time)
 848         and not line.is_import
 849         # there are no standalone comments in the body
 850         and not rhs.body.contains_standalone_comments(0)
 851         # and we can actually remove the parens
 852         and can_omit_invisible_parens(rhs, mode.line_length)
 853     ):
 854         omit = {id(rhs.closing_bracket), *omit}
 855         try:
 856             # The RHSResult Omitting Optional Parens.
 857             rhs_oop = _first_right_hand_split(line, omit=omit)
 858             if not (
 859                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 860                 # the split is right after `=`
 861                 and len(rhs.head.leaves) >= 2
 862                 and rhs.head.leaves[-2].type == token.EQUAL
 863                 # the left side of assignment contains brackets
 864                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 865                 # the left side of assignment is short enough (the -1 is for the ending
 866                 # optional paren)
 867                 and is_line_short_enough(
 868                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 869                 )
 870                 # the left side of assignment won't explode further because of magic
 871                 # trailing comma
 872                 and rhs.head.magic_trailing_comma is None
 873                 # the split by omitting optional parens isn't preferred by some other
 874                 # reason
 875                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 876             ):
 877                 yield from _maybe_split_omitting_optional_parens(
 878                     rhs_oop, line, mode, features=features, omit=omit
 879                 )
 880                 return
 881
 882         except CannotSplit as e:
 883             if not (
 884                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 885             ):
 886                 raise CannotSplit(
 887                     "Splitting failed, body is still too long and can't be split."
 888                 ) from e
 889
 890             elif (
 891                 rhs.head.contains_multiline_strings()
 892                 or rhs.tail.contains_multiline_strings()
 893             ):
 894                 raise CannotSplit(
 895                     "The current optional pair of parentheses is bound to fail to"
 896                     " satisfy the splitting algorithm because the head or the tail"
 897                     " contains multiline strings which by definition never fit one"
 898                     " line."
 899                 ) from e
 900
 901     ensure_visible(rhs.opening_bracket)
 902     ensure_visible(rhs.closing_bracket)
 903     for result in (rhs.head, rhs.body, rhs.tail):
 904         if result:
 905             yield result
 906
 907
 908 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 909     """
 910     Returns whether we should prefer the result from a split omitting optional parens.
 911     """
 912     has_closing_bracket_after_assign = False
 913     for leaf in reversed(rhs_oop.head.leaves):
 914         if leaf.type == token.EQUAL:
 915             break
 916         if leaf.type in CLOSING_BRACKETS:
 917             has_closing_bracket_after_assign = True
 918             break
 919     return (
 920         # contains matching brackets after the `=` (done by checking there is a
 921         # closing bracket)
 922         has_closing_bracket_after_assign
 923         or (
 924             # the split is actually from inside the optional parens (done by checking
 925             # the first line still contains the `=`)
 926             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 927             # the first line is short enough
 928             and is_line_short_enough(rhs_oop.head, mode=mode)
 929         )
 930         # contains unsplittable type ignore
 931         or rhs_oop.head.contains_unsplittable_type_ignore()
 932         or rhs_oop.body.contains_unsplittable_type_ignore()
 933         or rhs_oop.tail.contains_unsplittable_type_ignore()
 934     )
 935
 936
 937 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 938     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 939
 940     Do nothing otherwise.
 941
 942     A left- or right-hand split is based on a pair of brackets. Content before
 943     (and including) the opening bracket is left on one line, content inside the
 944     brackets is put on a separate line, and finally content starting with and
 945     following the closing bracket is put on a separate line.
 946
 947     Those are called `head`, `body`, and `tail`, respectively. If the split
 948     produced the same line (all content in `head`) or ended up with an empty `body`
 949     and the `tail` is just the closing bracket, then it's considered failed.
 950     """
 951     tail_len = len(str(tail).strip())
 952     if not body:
 953         if tail_len == 0:
 954             raise CannotSplit("Splitting brackets produced the same line")
 955
 956         elif tail_len < 3:
 957             raise CannotSplit(
 958                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 959                 " not worth it"
 960             )
 961
 962
 963 def bracket_split_build_line(
 964     leaves: List[Leaf],
 965     original: Line,
 966     opening_bracket: Leaf,
 967     *,
 968     component: _BracketSplitComponent,
 969 ) -> Line:
 970     """Return a new line with given `leaves` and respective comments from `original`.
 971
 972     If it's the head component, brackets will be tracked so trailing commas are
 973     respected.
 974
 975     If it's the body component, the result line is one-indented inside brackets and as
 976     such has its first leaf's prefix normalized and a trailing comma added when
 977     expected.
 978     """
 979     result = Line(mode=original.mode, depth=original.depth)
 980     if component is _BracketSplitComponent.body:
 981         result.inside_brackets = True
 982         result.depth += 1
 983         if leaves:
 984             # Since body is a new indent level, remove spurious leading whitespace.
 985             normalize_prefix(leaves[0], inside_brackets=True)
 986             # Ensure a trailing comma for imports and standalone function arguments, but
 987             # be careful not to add one after any comments or within type annotations.
 988             no_commas = (
 989                 original.is_def
 990                 and opening_bracket.value == "("
 991                 and not any(leaf.type == token.COMMA for leaf in leaves)
 992                 # In particular, don't add one within a parenthesized return annotation.
 993                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 994                 # be defined directly in the parent node, the parent of the parent ...
 995                 # and so on depending on how complex the return annotation is.
 996                 # This isn't perfect and there's some false negatives but they are in
 997                 # contexts were a comma is actually fine.
 998                 and not any(
 999                     node.prev_sibling.type == RARROW
1000                     for node in (
1001                         leaves[0].parent,
1002                         getattr(leaves[0].parent, "parent", None),
1003                     )
1004                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
1005                 )
1006                 # Except the false negatives above for PEP 604 unions where we
1007                 # can't add the comma.
1008                 and not (
1009                     leaves[0].parent
1010                     and leaves[0].parent.next_sibling
1011                     and leaves[0].parent.next_sibling.type == token.VBAR
1012                 )
1013             )
1014
1015             if original.is_import or no_commas:
1016                 for i in range(len(leaves) - 1, -1, -1):
1017                     if leaves[i].type == STANDALONE_COMMENT:
1018                         continue
1019
1020                     if leaves[i].type != token.COMMA:
1021                         new_comma = Leaf(token.COMMA, ",")
1022                         leaves.insert(i + 1, new_comma)
1023                     break
1024
1025     leaves_to_track: Set[LeafID] = set()
1026     if component is _BracketSplitComponent.head:
1027         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1028     # Populate the line
1029     for leaf in leaves:
1030         result.append(
1031             leaf,
1032             preformatted=True,
1033             track_bracket=id(leaf) in leaves_to_track,
1034         )
1035         for comment_after in original.comments_after(leaf):
1036             result.append(comment_after, preformatted=True)
1037     if component is _BracketSplitComponent.body and should_split_line(
1038         result, opening_bracket
1039     ):
1040         result.should_split_rhs = True
1041     return result
1042
1043
1044 def dont_increase_indentation(split_func: Transformer) -> Transformer:
1045     """Normalize prefix of the first leaf in every line returned by `split_func`.
1046
1047     This is a decorator over relevant split functions.
1048     """
1049
1050     @wraps(split_func)
1051     def split_wrapper(
1052         line: Line, features: Collection[Feature], mode: Mode
1053     ) -> Iterator[Line]:
1054         for split_line in split_func(line, features, mode):
1055             normalize_prefix(split_line.leaves[0], inside_brackets=True)
1056             yield split_line
1057
1058     return split_wrapper
1059
1060
1061 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1062     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1063         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1064             return leaf_idx
1065     return None
1066
1067
1068 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1069     if (
1070         safe
1071         and delimiter_priority == COMMA_PRIORITY
1072         and line.leaves[-1].type != token.COMMA
1073         and line.leaves[-1].type != STANDALONE_COMMENT
1074     ):
1075         new_comma = Leaf(token.COMMA, ",")
1076         line.append(new_comma)
1077     return line
1078
1079
1080 @dont_increase_indentation
1081 def delimiter_split(
1082     line: Line, features: Collection[Feature], mode: Mode
1083 ) -> Iterator[Line]:
1084     """Split according to delimiters of the highest priority.
1085
1086     If the appropriate Features are given, the split will add trailing commas
1087     also in function signatures and calls that contain `*` and `**`.
1088     """
1089     try:
1090         last_leaf = line.leaves[-1]
1091     except IndexError:
1092         raise CannotSplit("Line empty") from None
1093
1094     bt = line.bracket_tracker
1095     try:
1096         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1097     except ValueError:
1098         raise CannotSplit("No delimiters found") from None
1099
1100     if delimiter_priority == DOT_PRIORITY:
1101         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
1102             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1103
1104     current_line = Line(
1105         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1106     )
1107     lowest_depth = sys.maxsize
1108     trailing_comma_safe = True
1109
1110     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1111         """Append `leaf` to current line or to new line if appending impossible."""
1112         nonlocal current_line
1113         try:
1114             current_line.append_safe(leaf, preformatted=True)
1115         except ValueError:
1116             yield current_line
1117
1118             current_line = Line(
1119                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1120             )
1121             current_line.append(leaf)
1122
1123     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1124     for leaf_idx, leaf in enumerate(line.leaves):
1125         yield from append_to_line(leaf)
1126
1127         for comment_after in line.comments_after(leaf):
1128             yield from append_to_line(comment_after)
1129
1130         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1131         if leaf.bracket_depth == lowest_depth:
1132             if is_vararg(leaf, within={syms.typedargslist}):
1133                 trailing_comma_safe = (
1134                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1135                 )
1136             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1137                 trailing_comma_safe = (
1138                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1139                 )
1140
1141         if (
1142             Preview.add_trailing_comma_consistently in mode
1143             and last_leaf.type == STANDALONE_COMMENT
1144             and leaf_idx == last_non_comment_leaf
1145         ):
1146             current_line = _safe_add_trailing_comma(
1147                 trailing_comma_safe, delimiter_priority, current_line
1148             )
1149
1150         leaf_priority = bt.delimiters.get(id(leaf))
1151         if leaf_priority == delimiter_priority:
1152             yield current_line
1153
1154             current_line = Line(
1155                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1156             )
1157     if current_line:
1158         current_line = _safe_add_trailing_comma(
1159             trailing_comma_safe, delimiter_priority, current_line
1160         )
1161         yield current_line
1162
1163
1164 @dont_increase_indentation
1165 def standalone_comment_split(
1166     line: Line, features: Collection[Feature], mode: Mode
1167 ) -> Iterator[Line]:
1168     """Split standalone comments from the rest of the line."""
1169     if not line.contains_standalone_comments(0):
1170         raise CannotSplit("Line does not have any standalone comments")
1171
1172     current_line = Line(
1173         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1174     )
1175
1176     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1177         """Append `leaf` to current line or to new line if appending impossible."""
1178         nonlocal current_line
1179         try:
1180             current_line.append_safe(leaf, preformatted=True)
1181         except ValueError:
1182             yield current_line
1183
1184             current_line = Line(
1185                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1186             )
1187             current_line.append(leaf)
1188
1189     for leaf in line.leaves:
1190         yield from append_to_line(leaf)
1191
1192         for comment_after in line.comments_after(leaf):
1193             yield from append_to_line(comment_after)
1194
1195     if current_line:
1196         yield current_line
1197
1198
1199 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1200     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1201     else.
1202
1203     Note: don't use backslashes for formatting or you'll lose your voting rights.
1204     """
1205     if not inside_brackets:
1206         spl = leaf.prefix.split("#")
1207         if "\\" not in spl[0]:
1208             nl_count = spl[-1].count("\n")
1209             if len(spl) > 1:
1210                 nl_count -= 1
1211             leaf.prefix = "\n" * nl_count
1212             return
1213
1214     leaf.prefix = ""
1215
1216
1217 def normalize_invisible_parens(
1218     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1219 ) -> None:
1220     """Make existing optional parentheses invisible or create new ones.
1221
1222     `parens_after` is a set of string leaf values immediately after which parens
1223     should be put.
1224
1225     Standardizes on visible parentheses for single-element tuples, and keeps
1226     existing visible parentheses for other tuples and generator expressions.
1227     """
1228     for pc in list_comments(node.prefix, is_endmarker=False):
1229         if pc.value in FMT_OFF:
1230             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1231             return
1232
1233     # The multiple context managers grammar has a different pattern, thus this is
1234     # separate from the for-loop below. This possibly wraps them in invisible parens,
1235     # and later will be removed in remove_with_parens when needed.
1236     if node.type == syms.with_stmt:
1237         _maybe_wrap_cms_in_parens(node, mode, features)
1238
1239     check_lpar = False
1240     for index, child in enumerate(list(node.children)):
1241         # Fixes a bug where invisible parens are not properly stripped from
1242         # assignment statements that contain type annotations.
1243         if isinstance(child, Node) and child.type == syms.annassign:
1244             normalize_invisible_parens(
1245                 child, parens_after=parens_after, mode=mode, features=features
1246             )
1247
1248         # Add parentheses around long tuple unpacking in assignments.
1249         if (
1250             index == 0
1251             and isinstance(child, Node)
1252             and child.type == syms.testlist_star_expr
1253         ):
1254             check_lpar = True
1255
1256         if check_lpar:
1257             if (
1258                 child.type == syms.atom
1259                 and node.type == syms.for_stmt
1260                 and isinstance(child.prev_sibling, Leaf)
1261                 and child.prev_sibling.type == token.NAME
1262                 and child.prev_sibling.value == "for"
1263             ):
1264                 if maybe_make_parens_invisible_in_atom(
1265                     child,
1266                     parent=node,
1267                     remove_brackets_around_comma=True,
1268                 ):
1269                     wrap_in_parentheses(node, child, visible=False)
1270             elif isinstance(child, Node) and node.type == syms.with_stmt:
1271                 remove_with_parens(child, node)
1272             elif child.type == syms.atom:
1273                 if maybe_make_parens_invisible_in_atom(
1274                     child,
1275                     parent=node,
1276                 ):
1277                     wrap_in_parentheses(node, child, visible=False)
1278             elif is_one_tuple(child):
1279                 wrap_in_parentheses(node, child, visible=True)
1280             elif node.type == syms.import_from:
1281                 _normalize_import_from(node, child, index)
1282                 break
1283             elif (
1284                 index == 1
1285                 and child.type == token.STAR
1286                 and node.type == syms.except_clause
1287             ):
1288                 # In except* (PEP 654), the star is actually part of
1289                 # of the keyword. So we need to skip the insertion of
1290                 # invisible parentheses to work more precisely.
1291                 continue
1292
1293             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1294                 wrap_in_parentheses(node, child, visible=False)
1295
1296         comma_check = child.type == token.COMMA
1297
1298         check_lpar = isinstance(child, Leaf) and (
1299             child.value in parens_after or comma_check
1300         )
1301
1302
1303 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1304     # "import from" nodes store parentheses directly as part of
1305     # the statement
1306     if is_lpar_token(child):
1307         assert is_rpar_token(parent.children[-1])
1308         # make parentheses invisible
1309         child.value = ""
1310         parent.children[-1].value = ""
1311     elif child.type != token.STAR:
1312         # insert invisible parentheses
1313         parent.insert_child(index, Leaf(token.LPAR, ""))
1314         parent.append_child(Leaf(token.RPAR, ""))
1315
1316
1317 def remove_await_parens(node: Node) -> None:
1318     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1319         if (
1320             node.children[1].type == syms.atom
1321             and node.children[1].children[0].type == token.LPAR
1322         ):
1323             if maybe_make_parens_invisible_in_atom(
1324                 node.children[1],
1325                 parent=node,
1326                 remove_brackets_around_comma=True,
1327             ):
1328                 wrap_in_parentheses(node, node.children[1], visible=False)
1329
1330             # Since await is an expression we shouldn't remove
1331             # brackets in cases where this would change
1332             # the AST due to operator precedence.
1333             # Therefore we only aim to remove brackets around
1334             # power nodes that aren't also await expressions themselves.
1335             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1336             # N.B. We've still removed any redundant nested brackets though :)
1337             opening_bracket = cast(Leaf, node.children[1].children[0])
1338             closing_bracket = cast(Leaf, node.children[1].children[-1])
1339             bracket_contents = node.children[1].children[1]
1340             if isinstance(bracket_contents, Node):
1341                 if bracket_contents.type != syms.power:
1342                     ensure_visible(opening_bracket)
1343                     ensure_visible(closing_bracket)
1344                 elif (
1345                     bracket_contents.type == syms.power
1346                     and bracket_contents.children[0].type == token.AWAIT
1347                 ):
1348                     ensure_visible(opening_bracket)
1349                     ensure_visible(closing_bracket)
1350                     # If we are in a nested await then recurse down.
1351                     remove_await_parens(bracket_contents)
1352
1353
1354 def _maybe_wrap_cms_in_parens(
1355     node: Node, mode: Mode, features: Collection[Feature]
1356 ) -> None:
1357     """When enabled and safe, wrap the multiple context managers in invisible parens.
1358
1359     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1360     """
1361     if (
1362         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1363         or Preview.wrap_multiple_context_managers_in_parens not in mode
1364         or len(node.children) <= 2
1365         # If it's an atom, it's already wrapped in parens.
1366         or node.children[1].type == syms.atom
1367     ):
1368         return
1369     colon_index: Optional[int] = None
1370     for i in range(2, len(node.children)):
1371         if node.children[i].type == token.COLON:
1372             colon_index = i
1373             break
1374     if colon_index is not None:
1375         lpar = Leaf(token.LPAR, "")
1376         rpar = Leaf(token.RPAR, "")
1377         context_managers = node.children[1:colon_index]
1378         for child in context_managers:
1379             child.remove()
1380         # After wrapping, the with_stmt will look like this:
1381         #   with_stmt
1382         #     NAME 'with'
1383         #     atom
1384         #       LPAR ''
1385         #       testlist_gexp
1386         #         ... <-- context_managers
1387         #       /testlist_gexp
1388         #       RPAR ''
1389         #     /atom
1390         #     COLON ':'
1391         new_child = Node(
1392             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1393         )
1394         node.insert_child(1, new_child)
1395
1396
1397 def remove_with_parens(node: Node, parent: Node) -> None:
1398     """Recursively hide optional parens in `with` statements."""
1399     # Removing all unnecessary parentheses in with statements in one pass is a tad
1400     # complex as different variations of bracketed statements result in pretty
1401     # different parse trees:
1402     #
1403     # with (open("file")) as f:                       # this is an asexpr_test
1404     #     ...
1405     #
1406     # with (open("file") as f):                       # this is an atom containing an
1407     #     ...                                         # asexpr_test
1408     #
1409     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1410     #     ...                                         # asexpr_test
1411     #
1412     # with (open("file") as f, open("file") as f):    # an atom containing a
1413     #     ...                                         # testlist_gexp which then
1414     #                                                 # contains multiple asexpr_test(s)
1415     if node.type == syms.atom:
1416         if maybe_make_parens_invisible_in_atom(
1417             node,
1418             parent=parent,
1419             remove_brackets_around_comma=True,
1420         ):
1421             wrap_in_parentheses(parent, node, visible=False)
1422         if isinstance(node.children[1], Node):
1423             remove_with_parens(node.children[1], node)
1424     elif node.type == syms.testlist_gexp:
1425         for child in node.children:
1426             if isinstance(child, Node):
1427                 remove_with_parens(child, node)
1428     elif node.type == syms.asexpr_test and not any(
1429         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1430     ):
1431         if maybe_make_parens_invisible_in_atom(
1432             node.children[0],
1433             parent=node,
1434             remove_brackets_around_comma=True,
1435         ):
1436             wrap_in_parentheses(node, node.children[0], visible=False)
1437
1438
1439 def maybe_make_parens_invisible_in_atom(
1440     node: LN,
1441     parent: LN,
1442     remove_brackets_around_comma: bool = False,
1443 ) -> bool:
1444     """If it's safe, make the parens in the atom `node` invisible, recursively.
1445     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1446     as they are redundant.
1447
1448     Returns whether the node should itself be wrapped in invisible parentheses.
1449     """
1450     if (
1451         node.type not in (syms.atom, syms.expr)
1452         or is_empty_tuple(node)
1453         or is_one_tuple(node)
1454         or (is_yield(node) and parent.type != syms.expr_stmt)
1455         or (
1456             # This condition tries to prevent removing non-optional brackets
1457             # around a tuple, however, can be a bit overzealous so we provide
1458             # and option to skip this check for `for` and `with` statements.
1459             not remove_brackets_around_comma
1460             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1461         )
1462         or is_tuple_containing_walrus(node)
1463     ):
1464         return False
1465
1466     if is_walrus_assignment(node):
1467         if parent.type in [
1468             syms.annassign,
1469             syms.expr_stmt,
1470             syms.assert_stmt,
1471             syms.return_stmt,
1472             syms.except_clause,
1473             syms.funcdef,
1474             syms.with_stmt,
1475             syms.tname,
1476             # these ones aren't useful to end users, but they do please fuzzers
1477             syms.for_stmt,
1478             syms.del_stmt,
1479             syms.for_stmt,
1480         ]:
1481             return False
1482
1483     first = node.children[0]
1484     last = node.children[-1]
1485     if is_lpar_token(first) and is_rpar_token(last):
1486         middle = node.children[1]
1487         # make parentheses invisible
1488         if (
1489             # If the prefix of `middle` includes a type comment with
1490             # ignore annotation, then we do not remove the parentheses
1491             not is_type_ignore_comment_string(middle.prefix.strip())
1492         ):
1493             first.value = ""
1494             last.value = ""
1495         maybe_make_parens_invisible_in_atom(
1496             middle,
1497             parent=parent,
1498             remove_brackets_around_comma=remove_brackets_around_comma,
1499         )
1500
1501         if is_atom_with_invisible_parens(middle):
1502             # Strip the invisible parens from `middle` by replacing
1503             # it with the child in-between the invisible parens
1504             middle.replace(middle.children[1])
1505
1506         return False
1507
1508     return True
1509
1510
1511 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1512     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1513
1514     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1515         return False
1516
1517     # We're essentially checking if the body is delimited by commas and there's more
1518     # than one of them (we're excluding the trailing comma and if the delimiter priority
1519     # is still commas, that means there's more).
1520     exclude = set()
1521     trailing_comma = False
1522     try:
1523         last_leaf = line.leaves[-1]
1524         if last_leaf.type == token.COMMA:
1525             trailing_comma = True
1526             exclude.add(id(last_leaf))
1527         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1528     except (IndexError, ValueError):
1529         return False
1530
1531     return max_priority == COMMA_PRIORITY and (
1532         (line.mode.magic_trailing_comma and trailing_comma)
1533         # always explode imports
1534         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1535     )
1536
1537
1538 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1539     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1540
1541     Brackets can be omitted if the entire trailer up to and including
1542     a preceding closing bracket fits in one line.
1543
1544     Yielded sets are cumulative (contain results of previous yields, too).  First
1545     set is empty, unless the line should explode, in which case bracket pairs until
1546     the one that needs to explode are omitted.
1547     """
1548
1549     omit: Set[LeafID] = set()
1550     if not line.magic_trailing_comma:
1551         yield omit
1552
1553     length = 4 * line.depth
1554     opening_bracket: Optional[Leaf] = None
1555     closing_bracket: Optional[Leaf] = None
1556     inner_brackets: Set[LeafID] = set()
1557     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1558         length += leaf_length
1559         if length > line_length:
1560             break
1561
1562         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1563         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1564             break
1565
1566         if opening_bracket:
1567             if leaf is opening_bracket:
1568                 opening_bracket = None
1569             elif leaf.type in CLOSING_BRACKETS:
1570                 prev = line.leaves[index - 1] if index > 0 else None
1571                 if (
1572                     prev
1573                     and prev.type == token.COMMA
1574                     and leaf.opening_bracket is not None
1575                     and not is_one_sequence_between(
1576                         leaf.opening_bracket, leaf, line.leaves
1577                     )
1578                 ):
1579                     # Never omit bracket pairs with trailing commas.
1580                     # We need to explode on those.
1581                     break
1582
1583                 inner_brackets.add(id(leaf))
1584         elif leaf.type in CLOSING_BRACKETS:
1585             prev = line.leaves[index - 1] if index > 0 else None
1586             if prev and prev.type in OPENING_BRACKETS:
1587                 # Empty brackets would fail a split so treat them as "inner"
1588                 # brackets (e.g. only add them to the `omit` set if another
1589                 # pair of brackets was good enough.
1590                 inner_brackets.add(id(leaf))
1591                 continue
1592
1593             if closing_bracket:
1594                 omit.add(id(closing_bracket))
1595                 omit.update(inner_brackets)
1596                 inner_brackets.clear()
1597                 yield omit
1598
1599             if (
1600                 prev
1601                 and prev.type == token.COMMA
1602                 and leaf.opening_bracket is not None
1603                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1604             ):
1605                 # Never omit bracket pairs with trailing commas.
1606                 # We need to explode on those.
1607                 break
1608
1609             if leaf.value:
1610                 opening_bracket = leaf.opening_bracket
1611                 closing_bracket = leaf
1612
1613
1614 def run_transformer(
1615     line: Line,
1616     transform: Transformer,
1617     mode: Mode,
1618     features: Collection[Feature],
1619     *,
1620     line_str: str = "",
1621 ) -> List[Line]:
1622     if not line_str:
1623         line_str = line_to_string(line)
1624     result: List[Line] = []
1625     for transformed_line in transform(line, features, mode):
1626         if str(transformed_line).strip("\n") == line_str:
1627             raise CannotTransform("Line transformer returned an unchanged result")
1628
1629         result.extend(transform_line(transformed_line, mode=mode, features=features))
1630
1631     features_set = set(features)
1632     if (
1633         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1634         or transform.__class__.__name__ != "rhs"
1635         or not line.bracket_tracker.invisible
1636         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1637         or line.contains_multiline_strings()
1638         or result[0].contains_uncollapsable_type_comments()
1639         or result[0].contains_unsplittable_type_ignore()
1640         or is_line_short_enough(result[0], mode=mode)
1641         # If any leaves have no parents (which _can_ occur since
1642         # `transform(line)` potentially destroys the line's underlying node
1643         # structure), then we can't proceed. Doing so would cause the below
1644         # call to `append_leaves()` to fail.
1645         or any(leaf.parent is None for leaf in line.leaves)
1646     ):
1647         return result
1648
1649     line_copy = line.clone()
1650     append_leaves(line_copy, line, line.leaves)
1651     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1652     second_opinion = run_transformer(
1653         line_copy, transform, mode, features_fop, line_str=line_str
1654     )
1655     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1656         result = second_opinion
1657     return result