src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4
   5 import sys
   6 from dataclasses import replace
   7 from enum import Enum, auto
   8 from functools import partial, wraps
   9 from typing import Collection, Iterator, List, Optional, Set, Union, cast
  10
  11 from black.brackets import (
  12     COMMA_PRIORITY,
  13     DOT_PRIORITY,
  14     get_leaves_inside_matching_brackets,
  15     max_delimiter_priority_in_atom,
  16 )
  17 from black.comments import FMT_OFF, generate_comments, list_comments
  18 from black.lines import (
  19     Line,
  20     RHSResult,
  21     append_leaves,
  22     can_be_split,
  23     can_omit_invisible_parens,
  24     is_line_short_enough,
  25     line_to_string,
  26 )
  27 from black.mode import Feature, Mode, Preview
  28 from black.nodes import (
  29     ASSIGNMENTS,
  30     BRACKETS,
  31     CLOSING_BRACKETS,
  32     OPENING_BRACKETS,
  33     RARROW,
  34     STANDALONE_COMMENT,
  35     STATEMENT,
  36     WHITESPACE,
  37     Visitor,
  38     ensure_visible,
  39     is_arith_like,
  40     is_async_stmt_or_funcdef,
  41     is_atom_with_invisible_parens,
  42     is_docstring,
  43     is_empty_tuple,
  44     is_lpar_token,
  45     is_multiline_string,
  46     is_name_token,
  47     is_one_sequence_between,
  48     is_one_tuple,
  49     is_rpar_token,
  50     is_stub_body,
  51     is_stub_suite,
  52     is_tuple_containing_walrus,
  53     is_type_ignore_comment_string,
  54     is_vararg,
  55     is_walrus_assignment,
  56     is_yield,
  57     syms,
  58     wrap_in_parentheses,
  59 )
  60 from black.numerics import normalize_numeric_literal
  61 from black.strings import (
  62     fix_docstring,
  63     get_string_prefix,
  64     normalize_string_prefix,
  65     normalize_string_quotes,
  66     normalize_unicode_escape_sequences,
  67 )
  68 from black.trans import (
  69     CannotTransform,
  70     StringMerger,
  71     StringParenStripper,
  72     StringParenWrapper,
  73     StringSplitter,
  74     Transformer,
  75     hug_power_op,
  76 )
  77 from blib2to3.pgen2 import token
  78 from blib2to3.pytree import Leaf, Node
  79
  80 # types
  81 LeafID = int
  82 LN = Union[Leaf, Node]
  83
  84
  85 class CannotSplit(CannotTransform):
  86     """A readable split that fits the allotted line length is impossible."""
  87
  88
  89 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  90 # See also https://github.com/mypyc/mypyc/issues/827.
  91 class LineGenerator(Visitor[Line]):
  92     """Generates reformatted Line objects.  Empty lines are not emitted.
  93
  94     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  95     in ways that will no longer stringify to valid Python code on the tree.
  96     """
  97
  98     def __init__(self, mode: Mode, features: Collection[Feature]) -> None:
  99         self.mode = mode
 100         self.features = features
 101         self.current_line: Line
 102         self.__post_init__()
 103
 104     def line(self, indent: int = 0) -> Iterator[Line]:
 105         """Generate a line.
 106
 107         If the line is empty, only emit if it makes sense.
 108         If the line is too long, split it first and then generate.
 109
 110         If any lines were generated, set up a new current_line.
 111         """
 112         if not self.current_line:
 113             self.current_line.depth += indent
 114             return  # Line is empty, don't emit. Creating a new one unnecessary.
 115
 116         if (
 117             Preview.improved_async_statements_handling in self.mode
 118             and len(self.current_line.leaves) == 1
 119             and is_async_stmt_or_funcdef(self.current_line.leaves[0])
 120         ):
 121             # Special case for async def/for/with statements. `visit_async_stmt`
 122             # adds an `ASYNC` leaf then visits the child def/for/with statement
 123             # nodes. Line yields from those nodes shouldn't treat the former
 124             # `ASYNC` leaf as a complete line.
 125             return
 126
 127         complete_line = self.current_line
 128         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
 129         yield complete_line
 130
 131     def visit_default(self, node: LN) -> Iterator[Line]:
 132         """Default `visit_*()` implementation. Recurses to children of `node`."""
 133         if isinstance(node, Leaf):
 134             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
 135             for comment in generate_comments(node):
 136                 if any_open_brackets:
 137                     # any comment within brackets is subject to splitting
 138                     self.current_line.append(comment)
 139                 elif comment.type == token.COMMENT:
 140                     # regular trailing comment
 141                     self.current_line.append(comment)
 142                     yield from self.line()
 143
 144                 else:
 145                     # regular standalone comment
 146                     yield from self.line()
 147
 148                     self.current_line.append(comment)
 149                     yield from self.line()
 150
 151             normalize_prefix(node, inside_brackets=any_open_brackets)
 152             if self.mode.string_normalization and node.type == token.STRING:
 153                 node.value = normalize_string_prefix(node.value)
 154                 node.value = normalize_string_quotes(node.value)
 155             if node.type == token.NUMBER:
 156                 normalize_numeric_literal(node)
 157             if node.type not in WHITESPACE:
 158                 self.current_line.append(node)
 159         yield from super().visit_default(node)
 160
 161     def visit_test(self, node: Node) -> Iterator[Line]:
 162         """Visit an `x if y else z` test"""
 163
 164         if Preview.parenthesize_conditional_expressions in self.mode:
 165             already_parenthesized = (
 166                 node.prev_sibling and node.prev_sibling.type == token.LPAR
 167             )
 168
 169             if not already_parenthesized:
 170                 lpar = Leaf(token.LPAR, "")
 171                 rpar = Leaf(token.RPAR, "")
 172                 node.insert_child(0, lpar)
 173                 node.append_child(rpar)
 174
 175         yield from self.visit_default(node)
 176
 177     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 178         """Increase indentation level, maybe yield a line."""
 179         # In blib2to3 INDENT never holds comments.
 180         yield from self.line(+1)
 181         yield from self.visit_default(node)
 182
 183     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 184         """Decrease indentation level, maybe yield a line."""
 185         # The current line might still wait for trailing comments.  At DEDENT time
 186         # there won't be any (they would be prefixes on the preceding NEWLINE).
 187         # Emit the line then.
 188         yield from self.line()
 189
 190         # While DEDENT has no value, its prefix may contain standalone comments
 191         # that belong to the current indentation level.  Get 'em.
 192         yield from self.visit_default(node)
 193
 194         # Finally, emit the dedent.
 195         yield from self.line(-1)
 196
 197     def visit_stmt(
 198         self, node: Node, keywords: Set[str], parens: Set[str]
 199     ) -> Iterator[Line]:
 200         """Visit a statement.
 201
 202         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 203         `def`, `with`, `class`, `assert`, and assignments.
 204
 205         The relevant Python language `keywords` for a given statement will be
 206         NAME leaves within it. This methods puts those on a separate line.
 207
 208         `parens` holds a set of string leaf values immediately after which
 209         invisible parens should be put.
 210         """
 211         normalize_invisible_parens(
 212             node, parens_after=parens, mode=self.mode, features=self.features
 213         )
 214         for child in node.children:
 215             if is_name_token(child) and child.value in keywords:
 216                 yield from self.line()
 217
 218             yield from self.visit(child)
 219
 220     def visit_typeparams(self, node: Node) -> Iterator[Line]:
 221         yield from self.visit_default(node)
 222         node.children[0].prefix = ""
 223
 224     def visit_typevartuple(self, node: Node) -> Iterator[Line]:
 225         yield from self.visit_default(node)
 226         node.children[1].prefix = ""
 227
 228     def visit_paramspec(self, node: Node) -> Iterator[Line]:
 229         yield from self.visit_default(node)
 230         node.children[1].prefix = ""
 231
 232     def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:
 233         if Preview.wrap_long_dict_values_in_parens in self.mode:
 234             for i, child in enumerate(node.children):
 235                 if i == 0:
 236                     continue
 237                 if node.children[i - 1].type == token.COLON:
 238                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 239                         if maybe_make_parens_invisible_in_atom(
 240                             child,
 241                             parent=node,
 242                             remove_brackets_around_comma=False,
 243                         ):
 244                             wrap_in_parentheses(node, child, visible=False)
 245                     else:
 246                         wrap_in_parentheses(node, child, visible=False)
 247         yield from self.visit_default(node)
 248
 249     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 250         """Visit function definition."""
 251         yield from self.line()
 252
 253         # Remove redundant brackets around return type annotation.
 254         is_return_annotation = False
 255         for child in node.children:
 256             if child.type == token.RARROW:
 257                 is_return_annotation = True
 258             elif is_return_annotation:
 259                 if child.type == syms.atom and child.children[0].type == token.LPAR:
 260                     if maybe_make_parens_invisible_in_atom(
 261                         child,
 262                         parent=node,
 263                         remove_brackets_around_comma=False,
 264                     ):
 265                         wrap_in_parentheses(node, child, visible=False)
 266                 else:
 267                     wrap_in_parentheses(node, child, visible=False)
 268                 is_return_annotation = False
 269
 270         for child in node.children:
 271             yield from self.visit(child)
 272
 273     def visit_match_case(self, node: Node) -> Iterator[Line]:
 274         """Visit either a match or case statement."""
 275         normalize_invisible_parens(
 276             node, parens_after=set(), mode=self.mode, features=self.features
 277         )
 278
 279         yield from self.line()
 280         for child in node.children:
 281             yield from self.visit(child)
 282
 283     def visit_suite(self, node: Node) -> Iterator[Line]:
 284         """Visit a suite."""
 285         if (
 286             self.mode.is_pyi or Preview.dummy_implementations in self.mode
 287         ) and is_stub_suite(node):
 288             yield from self.visit(node.children[2])
 289         else:
 290             yield from self.visit_default(node)
 291
 292     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 293         """Visit a statement without nested statements."""
 294         prev_type: Optional[int] = None
 295         for child in node.children:
 296             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 297                 wrap_in_parentheses(node, child, visible=False)
 298             prev_type = child.type
 299
 300         is_suite_like = node.parent and node.parent.type in STATEMENT
 301         if is_suite_like:
 302             if (
 303                 self.mode.is_pyi or Preview.dummy_implementations in self.mode
 304             ) and is_stub_body(node):
 305                 yield from self.visit_default(node)
 306             else:
 307                 yield from self.line(+1)
 308                 yield from self.visit_default(node)
 309                 yield from self.line(-1)
 310
 311         else:
 312             if (
 313                 not (self.mode.is_pyi or Preview.dummy_implementations in self.mode)
 314                 or not node.parent
 315                 or not is_stub_suite(node.parent)
 316             ):
 317                 yield from self.line()
 318             yield from self.visit_default(node)
 319
 320     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 321         """Visit `async def`, `async for`, `async with`."""
 322         yield from self.line()
 323
 324         children = iter(node.children)
 325         for child in children:
 326             yield from self.visit(child)
 327
 328             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
 329                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
 330                 # line.
 331                 break
 332
 333         internal_stmt = next(children)
 334         if Preview.improved_async_statements_handling in self.mode:
 335             yield from self.visit(internal_stmt)
 336         else:
 337             for child in internal_stmt.children:
 338                 yield from self.visit(child)
 339
 340     def visit_decorators(self, node: Node) -> Iterator[Line]:
 341         """Visit decorators."""
 342         for child in node.children:
 343             yield from self.line()
 344             yield from self.visit(child)
 345
 346     def visit_power(self, node: Node) -> Iterator[Line]:
 347         for idx, leaf in enumerate(node.children[:-1]):
 348             next_leaf = node.children[idx + 1]
 349
 350             if not isinstance(leaf, Leaf):
 351                 continue
 352
 353             value = leaf.value.lower()
 354             if (
 355                 leaf.type == token.NUMBER
 356                 and next_leaf.type == syms.trailer
 357                 # Ensure that we are in an attribute trailer
 358                 and next_leaf.children[0].type == token.DOT
 359                 # It shouldn't wrap hexadecimal, binary and octal literals
 360                 and not value.startswith(("0x", "0b", "0o"))
 361                 # It shouldn't wrap complex literals
 362                 and "j" not in value
 363             ):
 364                 wrap_in_parentheses(node, leaf)
 365
 366         remove_await_parens(node)
 367
 368         yield from self.visit_default(node)
 369
 370     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 371         """Remove a semicolon and put the other statement on a separate line."""
 372         yield from self.line()
 373
 374     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 375         """End of file. Process outstanding comments and end with a newline."""
 376         yield from self.visit_default(leaf)
 377         yield from self.line()
 378
 379     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 380         if not self.current_line.bracket_tracker.any_open_brackets():
 381             yield from self.line()
 382         yield from self.visit_default(leaf)
 383
 384     def visit_factor(self, node: Node) -> Iterator[Line]:
 385         """Force parentheses between a unary op and a binary power:
 386
 387         -2 ** 8 -> -(2 ** 8)
 388         """
 389         _operator, operand = node.children
 390         if (
 391             operand.type == syms.power
 392             and len(operand.children) == 3
 393             and operand.children[1].type == token.DOUBLESTAR
 394         ):
 395             lpar = Leaf(token.LPAR, "(")
 396             rpar = Leaf(token.RPAR, ")")
 397             index = operand.remove() or 0
 398             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 399         yield from self.visit_default(node)
 400
 401     def visit_tname(self, node: Node) -> Iterator[Line]:
 402         """
 403         Add potential parentheses around types in function parameter lists to be made
 404         into real parentheses in case the type hint is too long to fit on a line
 405         Examples:
 406         def foo(a: int, b: float = 7): ...
 407
 408         ->
 409
 410         def foo(a: (int), b: (float) = 7): ...
 411         """
 412         if Preview.parenthesize_long_type_hints in self.mode:
 413             assert len(node.children) == 3
 414             if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):
 415                 wrap_in_parentheses(node, node.children[2], visible=False)
 416
 417         yield from self.visit_default(node)
 418
 419     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 420         if Preview.hex_codes_in_unicode_sequences in self.mode:
 421             normalize_unicode_escape_sequences(leaf)
 422
 423         if is_docstring(leaf) and "\\\n" not in leaf.value:
 424             # We're ignoring docstrings with backslash newline escapes because changing
 425             # indentation of those changes the AST representation of the code.
 426             if self.mode.string_normalization:
 427                 docstring = normalize_string_prefix(leaf.value)
 428                 # visit_default() does handle string normalization for us, but
 429                 # since this method acts differently depending on quote style (ex.
 430                 # see padding logic below), there's a possibility for unstable
 431                 # formatting as visit_default() is called *after*. To avoid a
 432                 # situation where this function formats a docstring differently on
 433                 # the second pass, normalize it early.
 434                 docstring = normalize_string_quotes(docstring)
 435             else:
 436                 docstring = leaf.value
 437             prefix = get_string_prefix(docstring)
 438             docstring = docstring[len(prefix) :]  # Remove the prefix
 439             quote_char = docstring[0]
 440             # A natural way to remove the outer quotes is to do:
 441             #   docstring = docstring.strip(quote_char)
 442             # but that breaks on """""x""" (which is '""x').
 443             # So we actually need to remove the first character and the next two
 444             # characters but only if they are the same as the first.
 445             quote_len = 1 if docstring[1] != quote_char else 3
 446             docstring = docstring[quote_len:-quote_len]
 447             docstring_started_empty = not docstring
 448             indent = " " * 4 * self.current_line.depth
 449
 450             if is_multiline_string(leaf):
 451                 docstring = fix_docstring(docstring, indent)
 452             else:
 453                 docstring = docstring.strip()
 454
 455             has_trailing_backslash = False
 456             if docstring:
 457                 # Add some padding if the docstring starts / ends with a quote mark.
 458                 if docstring[0] == quote_char:
 459                     docstring = " " + docstring
 460                 if docstring[-1] == quote_char:
 461                     docstring += " "
 462                 if docstring[-1] == "\\":
 463                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 464                     if backslash_count % 2:
 465                         # Odd number of tailing backslashes, add some padding to
 466                         # avoid escaping the closing string quote.
 467                         docstring += " "
 468                         has_trailing_backslash = True
 469             elif not docstring_started_empty:
 470                 docstring = " "
 471
 472             # We could enforce triple quotes at this point.
 473             quote = quote_char * quote_len
 474
 475             # It's invalid to put closing single-character quotes on a new line.
 476             if self.mode and quote_len == 3:
 477                 # We need to find the length of the last line of the docstring
 478                 # to find if we can add the closing quotes to the line without
 479                 # exceeding the maximum line length.
 480                 # If docstring is one line, we don't put the closing quotes on a
 481                 # separate line because it looks ugly (#3320).
 482                 lines = docstring.splitlines()
 483                 last_line_length = len(lines[-1]) if docstring else 0
 484
 485                 # If adding closing quotes would cause the last line to exceed
 486                 # the maximum line length then put a line break before the
 487                 # closing quotes
 488                 if (
 489                     len(lines) > 1
 490                     and last_line_length + quote_len > self.mode.line_length
 491                     and len(indent) + quote_len <= self.mode.line_length
 492                     and not has_trailing_backslash
 493                 ):
 494                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 495                 else:
 496                     leaf.value = prefix + quote + docstring + quote
 497             else:
 498                 leaf.value = prefix + quote + docstring + quote
 499
 500         yield from self.visit_default(leaf)
 501
 502     def __post_init__(self) -> None:
 503         """You are in a twisty little maze of passages."""
 504         self.current_line = Line(mode=self.mode)
 505
 506         v = self.visit_stmt
 507         Ø: Set[str] = set()
 508         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 509         self.visit_if_stmt = partial(
 510             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 511         )
 512         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 513         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 514         self.visit_try_stmt = partial(
 515             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 516         )
 517         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
 518         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 519         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 520
 521         # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py
 522         if Preview.parenthesize_long_type_hints in self.mode:
 523             assignments = ASSIGNMENTS | {":"}
 524         else:
 525             assignments = ASSIGNMENTS
 526         self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments)
 527
 528         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 529         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 530         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 531         self.visit_async_funcdef = self.visit_async_stmt
 532         self.visit_decorated = self.visit_decorators
 533
 534         # PEP 634
 535         self.visit_match_stmt = self.visit_match_case
 536         self.visit_case_block = self.visit_match_case
 537
 538
 539 def transform_line(
 540     line: Line, mode: Mode, features: Collection[Feature] = ()
 541 ) -> Iterator[Line]:
 542     """Transform a `line`, potentially splitting it into many lines.
 543
 544     They should fit in the allotted `line_length` but might not be able to.
 545
 546     `features` are syntactical features that may be used in the output.
 547     """
 548     if line.is_comment:
 549         yield line
 550         return
 551
 552     line_str = line_to_string(line)
 553
 554     ll = mode.line_length
 555     sn = mode.string_normalization
 556     string_merge = StringMerger(ll, sn)
 557     string_paren_strip = StringParenStripper(ll, sn)
 558     string_split = StringSplitter(ll, sn)
 559     string_paren_wrap = StringParenWrapper(ll, sn)
 560
 561     transformers: List[Transformer]
 562     if (
 563         not line.contains_uncollapsable_type_comments()
 564         and not line.should_split_rhs
 565         and not line.magic_trailing_comma
 566         and (
 567             is_line_short_enough(line, mode=mode, line_str=line_str)
 568             or line.contains_unsplittable_type_ignore()
 569         )
 570         and not (line.inside_brackets and line.contains_standalone_comments())
 571     ):
 572         # Only apply basic string preprocessing, since lines shouldn't be split here.
 573         if Preview.string_processing in mode:
 574             transformers = [string_merge, string_paren_strip]
 575         else:
 576             transformers = []
 577     elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
 578         transformers = [left_hand_split]
 579     else:
 580
 581         def _rhs(
 582             self: object, line: Line, features: Collection[Feature], mode: Mode
 583         ) -> Iterator[Line]:
 584             """Wraps calls to `right_hand_split`.
 585
 586             The calls increasingly `omit` right-hand trailers (bracket pairs with
 587             content), meaning the trailers get glued together to split on another
 588             bracket pair instead.
 589             """
 590             for omit in generate_trailers_to_omit(line, mode.line_length):
 591                 lines = list(right_hand_split(line, mode, features, omit=omit))
 592                 # Note: this check is only able to figure out if the first line of the
 593                 # *current* transformation fits in the line length.  This is true only
 594                 # for simple cases.  All others require running more transforms via
 595                 # `transform_line()`.  This check doesn't know if those would succeed.
 596                 if is_line_short_enough(lines[0], mode=mode):
 597                     yield from lines
 598                     return
 599
 600             # All splits failed, best effort split with no omits.
 601             # This mostly happens to multiline strings that are by definition
 602             # reported as not fitting a single line, as well as lines that contain
 603             # trailing commas (those have to be exploded).
 604             yield from right_hand_split(line, mode, features=features)
 605
 606         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 607         # __name__ attribute which is needed in `run_transformer` further down.
 608         # Unfortunately a nested class breaks mypyc too. So a class must be created
 609         # via type ... https://github.com/mypyc/mypyc/issues/884
 610         rhs = type("rhs", (), {"__call__": _rhs})()
 611
 612         if Preview.string_processing in mode:
 613             if line.inside_brackets:
 614                 transformers = [
 615                     string_merge,
 616                     string_paren_strip,
 617                     string_split,
 618                     delimiter_split,
 619                     standalone_comment_split,
 620                     string_paren_wrap,
 621                     rhs,
 622                 ]
 623             else:
 624                 transformers = [
 625                     string_merge,
 626                     string_paren_strip,
 627                     string_split,
 628                     string_paren_wrap,
 629                     rhs,
 630                 ]
 631         else:
 632             if line.inside_brackets:
 633                 transformers = [delimiter_split, standalone_comment_split, rhs]
 634             else:
 635                 transformers = [rhs]
 636     # It's always safe to attempt hugging of power operations and pretty much every line
 637     # could match.
 638     transformers.append(hug_power_op)
 639
 640     for transform in transformers:
 641         # We are accumulating lines in `result` because we might want to abort
 642         # mission and return the original line in the end, or attempt a different
 643         # split altogether.
 644         try:
 645             result = run_transformer(line, transform, mode, features, line_str=line_str)
 646         except CannotTransform:
 647             continue
 648         else:
 649             yield from result
 650             break
 651
 652     else:
 653         yield line
 654
 655
 656 def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
 657     """If a funcdef has a magic trailing comma in the return type, then we should first
 658     split the line with rhs to respect the comma.
 659     """
 660     if Preview.respect_magic_trailing_comma_in_return_type not in mode:
 661         return False
 662
 663     return_type_leaves: List[Leaf] = []
 664     in_return_type = False
 665
 666     for leaf in line.leaves:
 667         if leaf.type == token.COLON:
 668             in_return_type = False
 669         if in_return_type:
 670             return_type_leaves.append(leaf)
 671         if leaf.type == token.RARROW:
 672             in_return_type = True
 673
 674     # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
 675     # couple lines from it.
 676     result = Line(mode=line.mode, depth=line.depth)
 677     leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
 678     for leaf in return_type_leaves:
 679         result.append(
 680             leaf,
 681             preformatted=True,
 682             track_bracket=id(leaf) in leaves_to_track,
 683         )
 684
 685     # we could also return true if the line is too long, and the return type is longer
 686     # than the param list. Or if `should_split_rhs` returns True.
 687     return result.magic_trailing_comma is not None
 688
 689
 690 class _BracketSplitComponent(Enum):
 691     head = auto()
 692     body = auto()
 693     tail = auto()
 694
 695
 696 def left_hand_split(
 697     line: Line, _features: Collection[Feature], mode: Mode
 698 ) -> Iterator[Line]:
 699     """Split line into many lines, starting with the first matching bracket pair.
 700
 701     Note: this usually looks weird, only use this for function definitions.
 702     Prefer RHS otherwise.  This is why this function is not symmetrical with
 703     :func:`right_hand_split` which also handles optional parentheses.
 704     """
 705     tail_leaves: List[Leaf] = []
 706     body_leaves: List[Leaf] = []
 707     head_leaves: List[Leaf] = []
 708     current_leaves = head_leaves
 709     matching_bracket: Optional[Leaf] = None
 710     for leaf in line.leaves:
 711         if (
 712             current_leaves is body_leaves
 713             and leaf.type in CLOSING_BRACKETS
 714             and leaf.opening_bracket is matching_bracket
 715             and isinstance(matching_bracket, Leaf)
 716         ):
 717             ensure_visible(leaf)
 718             ensure_visible(matching_bracket)
 719             current_leaves = tail_leaves if body_leaves else head_leaves
 720         current_leaves.append(leaf)
 721         if current_leaves is head_leaves:
 722             if leaf.type in OPENING_BRACKETS:
 723                 matching_bracket = leaf
 724                 current_leaves = body_leaves
 725     if not matching_bracket:
 726         raise CannotSplit("No brackets found")
 727
 728     head = bracket_split_build_line(
 729         head_leaves, line, matching_bracket, component=_BracketSplitComponent.head
 730     )
 731     body = bracket_split_build_line(
 732         body_leaves, line, matching_bracket, component=_BracketSplitComponent.body
 733     )
 734     tail = bracket_split_build_line(
 735         tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail
 736     )
 737     bracket_split_succeeded_or_raise(head, body, tail)
 738     for result in (head, body, tail):
 739         if result:
 740             yield result
 741
 742
 743 def right_hand_split(
 744     line: Line,
 745     mode: Mode,
 746     features: Collection[Feature] = (),
 747     omit: Collection[LeafID] = (),
 748 ) -> Iterator[Line]:
 749     """Split line into many lines, starting with the last matching bracket pair.
 750
 751     If the split was by optional parentheses, attempt splitting without them, too.
 752     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 753     this split.
 754
 755     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 756     """
 757     rhs_result = _first_right_hand_split(line, omit=omit)
 758     yield from _maybe_split_omitting_optional_parens(
 759         rhs_result, line, mode, features=features, omit=omit
 760     )
 761
 762
 763 def _first_right_hand_split(
 764     line: Line,
 765     omit: Collection[LeafID] = (),
 766 ) -> RHSResult:
 767     """Split the line into head, body, tail starting with the last bracket pair.
 768
 769     Note: this function should not have side effects. It's relied upon by
 770     _maybe_split_omitting_optional_parens to get an opinion whether to prefer
 771     splitting on the right side of an assignment statement.
 772     """
 773     tail_leaves: List[Leaf] = []
 774     body_leaves: List[Leaf] = []
 775     head_leaves: List[Leaf] = []
 776     current_leaves = tail_leaves
 777     opening_bracket: Optional[Leaf] = None
 778     closing_bracket: Optional[Leaf] = None
 779     for leaf in reversed(line.leaves):
 780         if current_leaves is body_leaves:
 781             if leaf is opening_bracket:
 782                 current_leaves = head_leaves if body_leaves else tail_leaves
 783         current_leaves.append(leaf)
 784         if current_leaves is tail_leaves:
 785             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 786                 opening_bracket = leaf.opening_bracket
 787                 closing_bracket = leaf
 788                 current_leaves = body_leaves
 789     if not (opening_bracket and closing_bracket and head_leaves):
 790         # If there is no opening or closing_bracket that means the split failed and
 791         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 792         # the matching `opening_bracket` wasn't available on `line` anymore.
 793         raise CannotSplit("No brackets found")
 794
 795     tail_leaves.reverse()
 796     body_leaves.reverse()
 797     head_leaves.reverse()
 798     head = bracket_split_build_line(
 799         head_leaves, line, opening_bracket, component=_BracketSplitComponent.head
 800     )
 801     body = bracket_split_build_line(
 802         body_leaves, line, opening_bracket, component=_BracketSplitComponent.body
 803     )
 804     tail = bracket_split_build_line(
 805         tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail
 806     )
 807     bracket_split_succeeded_or_raise(head, body, tail)
 808     return RHSResult(head, body, tail, opening_bracket, closing_bracket)
 809
 810
 811 def _maybe_split_omitting_optional_parens(
 812     rhs: RHSResult,
 813     line: Line,
 814     mode: Mode,
 815     features: Collection[Feature] = (),
 816     omit: Collection[LeafID] = (),
 817 ) -> Iterator[Line]:
 818     if (
 819         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 820         # the opening bracket is an optional paren
 821         and rhs.opening_bracket.type == token.LPAR
 822         and not rhs.opening_bracket.value
 823         # the closing bracket is an optional paren
 824         and rhs.closing_bracket.type == token.RPAR
 825         and not rhs.closing_bracket.value
 826         # it's not an import (optional parens are the only thing we can split on
 827         # in this case; attempting a split without them is a waste of time)
 828         and not line.is_import
 829         # there are no standalone comments in the body
 830         and not rhs.body.contains_standalone_comments(0)
 831         # and we can actually remove the parens
 832         and can_omit_invisible_parens(rhs, mode.line_length)
 833     ):
 834         omit = {id(rhs.closing_bracket), *omit}
 835         try:
 836             # The RHSResult Omitting Optional Parens.
 837             rhs_oop = _first_right_hand_split(line, omit=omit)
 838             if not (
 839                 Preview.prefer_splitting_right_hand_side_of_assignments in line.mode
 840                 # the split is right after `=`
 841                 and len(rhs.head.leaves) >= 2
 842                 and rhs.head.leaves[-2].type == token.EQUAL
 843                 # the left side of assignment contains brackets
 844                 and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1])
 845                 # the left side of assignment is short enough (the -1 is for the ending
 846                 # optional paren)
 847                 and is_line_short_enough(
 848                     rhs.head, mode=replace(mode, line_length=mode.line_length - 1)
 849                 )
 850                 # the left side of assignment won't explode further because of magic
 851                 # trailing comma
 852                 and rhs.head.magic_trailing_comma is None
 853                 # the split by omitting optional parens isn't preferred by some other
 854                 # reason
 855                 and not _prefer_split_rhs_oop(rhs_oop, mode)
 856             ):
 857                 yield from _maybe_split_omitting_optional_parens(
 858                     rhs_oop, line, mode, features=features, omit=omit
 859                 )
 860                 return
 861
 862         except CannotSplit as e:
 863             if not (
 864                 can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode)
 865             ):
 866                 raise CannotSplit(
 867                     "Splitting failed, body is still too long and can't be split."
 868                 ) from e
 869
 870             elif (
 871                 rhs.head.contains_multiline_strings()
 872                 or rhs.tail.contains_multiline_strings()
 873             ):
 874                 raise CannotSplit(
 875                     "The current optional pair of parentheses is bound to fail to"
 876                     " satisfy the splitting algorithm because the head or the tail"
 877                     " contains multiline strings which by definition never fit one"
 878                     " line."
 879                 ) from e
 880
 881     ensure_visible(rhs.opening_bracket)
 882     ensure_visible(rhs.closing_bracket)
 883     for result in (rhs.head, rhs.body, rhs.tail):
 884         if result:
 885             yield result
 886
 887
 888 def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool:
 889     """
 890     Returns whether we should prefer the result from a split omitting optional parens.
 891     """
 892     has_closing_bracket_after_assign = False
 893     for leaf in reversed(rhs_oop.head.leaves):
 894         if leaf.type == token.EQUAL:
 895             break
 896         if leaf.type in CLOSING_BRACKETS:
 897             has_closing_bracket_after_assign = True
 898             break
 899     return (
 900         # contains matching brackets after the `=` (done by checking there is a
 901         # closing bracket)
 902         has_closing_bracket_after_assign
 903         or (
 904             # the split is actually from inside the optional parens (done by checking
 905             # the first line still contains the `=`)
 906             any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves)
 907             # the first line is short enough
 908             and is_line_short_enough(rhs_oop.head, mode=mode)
 909         )
 910         # contains unsplittable type ignore
 911         or rhs_oop.head.contains_unsplittable_type_ignore()
 912         or rhs_oop.body.contains_unsplittable_type_ignore()
 913         or rhs_oop.tail.contains_unsplittable_type_ignore()
 914     )
 915
 916
 917 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 918     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 919
 920     Do nothing otherwise.
 921
 922     A left- or right-hand split is based on a pair of brackets. Content before
 923     (and including) the opening bracket is left on one line, content inside the
 924     brackets is put on a separate line, and finally content starting with and
 925     following the closing bracket is put on a separate line.
 926
 927     Those are called `head`, `body`, and `tail`, respectively. If the split
 928     produced the same line (all content in `head`) or ended up with an empty `body`
 929     and the `tail` is just the closing bracket, then it's considered failed.
 930     """
 931     tail_len = len(str(tail).strip())
 932     if not body:
 933         if tail_len == 0:
 934             raise CannotSplit("Splitting brackets produced the same line")
 935
 936         elif tail_len < 3:
 937             raise CannotSplit(
 938                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 939                 " not worth it"
 940             )
 941
 942
 943 def bracket_split_build_line(
 944     leaves: List[Leaf],
 945     original: Line,
 946     opening_bracket: Leaf,
 947     *,
 948     component: _BracketSplitComponent,
 949 ) -> Line:
 950     """Return a new line with given `leaves` and respective comments from `original`.
 951
 952     If it's the head component, brackets will be tracked so trailing commas are
 953     respected.
 954
 955     If it's the body component, the result line is one-indented inside brackets and as
 956     such has its first leaf's prefix normalized and a trailing comma added when
 957     expected.
 958     """
 959     result = Line(mode=original.mode, depth=original.depth)
 960     if component is _BracketSplitComponent.body:
 961         result.inside_brackets = True
 962         result.depth += 1
 963         if leaves:
 964             # Since body is a new indent level, remove spurious leading whitespace.
 965             normalize_prefix(leaves[0], inside_brackets=True)
 966             # Ensure a trailing comma for imports and standalone function arguments, but
 967             # be careful not to add one after any comments or within type annotations.
 968             no_commas = (
 969                 original.is_def
 970                 and opening_bracket.value == "("
 971                 and not any(leaf.type == token.COMMA for leaf in leaves)
 972                 # In particular, don't add one within a parenthesized return annotation.
 973                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 974                 # be defined directly in the parent node, the parent of the parent ...
 975                 # and so on depending on how complex the return annotation is.
 976                 # This isn't perfect and there's some false negatives but they are in
 977                 # contexts were a comma is actually fine.
 978                 and not any(
 979                     node.prev_sibling.type == RARROW
 980                     for node in (
 981                         leaves[0].parent,
 982                         getattr(leaves[0].parent, "parent", None),
 983                     )
 984                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 985                 )
 986                 # Except the false negatives above for PEP 604 unions where we
 987                 # can't add the comma.
 988                 and not (
 989                     leaves[0].parent
 990                     and leaves[0].parent.next_sibling
 991                     and leaves[0].parent.next_sibling.type == token.VBAR
 992                 )
 993             )
 994
 995             if original.is_import or no_commas:
 996                 for i in range(len(leaves) - 1, -1, -1):
 997                     if leaves[i].type == STANDALONE_COMMENT:
 998                         continue
 999
1000                     if leaves[i].type != token.COMMA:
1001                         new_comma = Leaf(token.COMMA, ",")
1002                         leaves.insert(i + 1, new_comma)
1003                     break
1004
1005     leaves_to_track: Set[LeafID] = set()
1006     if component is _BracketSplitComponent.head:
1007         leaves_to_track = get_leaves_inside_matching_brackets(leaves)
1008     # Populate the line
1009     for leaf in leaves:
1010         result.append(
1011             leaf,
1012             preformatted=True,
1013             track_bracket=id(leaf) in leaves_to_track,
1014         )
1015         for comment_after in original.comments_after(leaf):
1016             result.append(comment_after, preformatted=True)
1017     if component is _BracketSplitComponent.body and should_split_line(
1018         result, opening_bracket
1019     ):
1020         result.should_split_rhs = True
1021     return result
1022
1023
1024 def dont_increase_indentation(split_func: Transformer) -> Transformer:
1025     """Normalize prefix of the first leaf in every line returned by `split_func`.
1026
1027     This is a decorator over relevant split functions.
1028     """
1029
1030     @wraps(split_func)
1031     def split_wrapper(
1032         line: Line, features: Collection[Feature], mode: Mode
1033     ) -> Iterator[Line]:
1034         for split_line in split_func(line, features, mode):
1035             normalize_prefix(split_line.leaves[0], inside_brackets=True)
1036             yield split_line
1037
1038     return split_wrapper
1039
1040
1041 def _get_last_non_comment_leaf(line: Line) -> Optional[int]:
1042     for leaf_idx in range(len(line.leaves) - 1, 0, -1):
1043         if line.leaves[leaf_idx].type != STANDALONE_COMMENT:
1044             return leaf_idx
1045     return None
1046
1047
1048 def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line:
1049     if (
1050         safe
1051         and delimiter_priority == COMMA_PRIORITY
1052         and line.leaves[-1].type != token.COMMA
1053         and line.leaves[-1].type != STANDALONE_COMMENT
1054     ):
1055         new_comma = Leaf(token.COMMA, ",")
1056         line.append(new_comma)
1057     return line
1058
1059
1060 @dont_increase_indentation
1061 def delimiter_split(
1062     line: Line, features: Collection[Feature], mode: Mode
1063 ) -> Iterator[Line]:
1064     """Split according to delimiters of the highest priority.
1065
1066     If the appropriate Features are given, the split will add trailing commas
1067     also in function signatures and calls that contain `*` and `**`.
1068     """
1069     try:
1070         last_leaf = line.leaves[-1]
1071     except IndexError:
1072         raise CannotSplit("Line empty") from None
1073
1074     bt = line.bracket_tracker
1075     try:
1076         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
1077     except ValueError:
1078         raise CannotSplit("No delimiters found") from None
1079
1080     if delimiter_priority == DOT_PRIORITY:
1081         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
1082             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
1083
1084     current_line = Line(
1085         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1086     )
1087     lowest_depth = sys.maxsize
1088     trailing_comma_safe = True
1089
1090     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1091         """Append `leaf` to current line or to new line if appending impossible."""
1092         nonlocal current_line
1093         try:
1094             current_line.append_safe(leaf, preformatted=True)
1095         except ValueError:
1096             yield current_line
1097
1098             current_line = Line(
1099                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1100             )
1101             current_line.append(leaf)
1102
1103     last_non_comment_leaf = _get_last_non_comment_leaf(line)
1104     for leaf_idx, leaf in enumerate(line.leaves):
1105         yield from append_to_line(leaf)
1106
1107         for comment_after in line.comments_after(leaf):
1108             yield from append_to_line(comment_after)
1109
1110         lowest_depth = min(lowest_depth, leaf.bracket_depth)
1111         if leaf.bracket_depth == lowest_depth:
1112             if is_vararg(leaf, within={syms.typedargslist}):
1113                 trailing_comma_safe = (
1114                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
1115                 )
1116             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
1117                 trailing_comma_safe = (
1118                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
1119                 )
1120
1121         if (
1122             Preview.add_trailing_comma_consistently in mode
1123             and last_leaf.type == STANDALONE_COMMENT
1124             and leaf_idx == last_non_comment_leaf
1125         ):
1126             current_line = _safe_add_trailing_comma(
1127                 trailing_comma_safe, delimiter_priority, current_line
1128             )
1129
1130         leaf_priority = bt.delimiters.get(id(leaf))
1131         if leaf_priority == delimiter_priority:
1132             yield current_line
1133
1134             current_line = Line(
1135                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1136             )
1137     if current_line:
1138         current_line = _safe_add_trailing_comma(
1139             trailing_comma_safe, delimiter_priority, current_line
1140         )
1141         yield current_line
1142
1143
1144 @dont_increase_indentation
1145 def standalone_comment_split(
1146     line: Line, features: Collection[Feature], mode: Mode
1147 ) -> Iterator[Line]:
1148     """Split standalone comments from the rest of the line."""
1149     if not line.contains_standalone_comments(0):
1150         raise CannotSplit("Line does not have any standalone comments")
1151
1152     current_line = Line(
1153         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1154     )
1155
1156     def append_to_line(leaf: Leaf) -> Iterator[Line]:
1157         """Append `leaf` to current line or to new line if appending impossible."""
1158         nonlocal current_line
1159         try:
1160             current_line.append_safe(leaf, preformatted=True)
1161         except ValueError:
1162             yield current_line
1163
1164             current_line = Line(
1165                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
1166             )
1167             current_line.append(leaf)
1168
1169     for leaf in line.leaves:
1170         yield from append_to_line(leaf)
1171
1172         for comment_after in line.comments_after(leaf):
1173             yield from append_to_line(comment_after)
1174
1175     if current_line:
1176         yield current_line
1177
1178
1179 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
1180     """Leave existing extra newlines if not `inside_brackets`. Remove everything
1181     else.
1182
1183     Note: don't use backslashes for formatting or you'll lose your voting rights.
1184     """
1185     if not inside_brackets:
1186         spl = leaf.prefix.split("#")
1187         if "\\" not in spl[0]:
1188             nl_count = spl[-1].count("\n")
1189             if len(spl) > 1:
1190                 nl_count -= 1
1191             leaf.prefix = "\n" * nl_count
1192             return
1193
1194     leaf.prefix = ""
1195
1196
1197 def normalize_invisible_parens(
1198     node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]
1199 ) -> None:
1200     """Make existing optional parentheses invisible or create new ones.
1201
1202     `parens_after` is a set of string leaf values immediately after which parens
1203     should be put.
1204
1205     Standardizes on visible parentheses for single-element tuples, and keeps
1206     existing visible parentheses for other tuples and generator expressions.
1207     """
1208     for pc in list_comments(node.prefix, is_endmarker=False):
1209         if pc.value in FMT_OFF:
1210             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
1211             return
1212
1213     # The multiple context managers grammar has a different pattern, thus this is
1214     # separate from the for-loop below. This possibly wraps them in invisible parens,
1215     # and later will be removed in remove_with_parens when needed.
1216     if node.type == syms.with_stmt:
1217         _maybe_wrap_cms_in_parens(node, mode, features)
1218
1219     check_lpar = False
1220     for index, child in enumerate(list(node.children)):
1221         # Fixes a bug where invisible parens are not properly stripped from
1222         # assignment statements that contain type annotations.
1223         if isinstance(child, Node) and child.type == syms.annassign:
1224             normalize_invisible_parens(
1225                 child, parens_after=parens_after, mode=mode, features=features
1226             )
1227
1228         # Add parentheses around long tuple unpacking in assignments.
1229         if (
1230             index == 0
1231             and isinstance(child, Node)
1232             and child.type == syms.testlist_star_expr
1233         ):
1234             check_lpar = True
1235
1236         if check_lpar:
1237             if (
1238                 child.type == syms.atom
1239                 and node.type == syms.for_stmt
1240                 and isinstance(child.prev_sibling, Leaf)
1241                 and child.prev_sibling.type == token.NAME
1242                 and child.prev_sibling.value == "for"
1243             ):
1244                 if maybe_make_parens_invisible_in_atom(
1245                     child,
1246                     parent=node,
1247                     remove_brackets_around_comma=True,
1248                 ):
1249                     wrap_in_parentheses(node, child, visible=False)
1250             elif isinstance(child, Node) and node.type == syms.with_stmt:
1251                 remove_with_parens(child, node)
1252             elif child.type == syms.atom:
1253                 if maybe_make_parens_invisible_in_atom(
1254                     child,
1255                     parent=node,
1256                 ):
1257                     wrap_in_parentheses(node, child, visible=False)
1258             elif is_one_tuple(child):
1259                 wrap_in_parentheses(node, child, visible=True)
1260             elif node.type == syms.import_from:
1261                 _normalize_import_from(node, child, index)
1262                 break
1263             elif (
1264                 index == 1
1265                 and child.type == token.STAR
1266                 and node.type == syms.except_clause
1267             ):
1268                 # In except* (PEP 654), the star is actually part of
1269                 # of the keyword. So we need to skip the insertion of
1270                 # invisible parentheses to work more precisely.
1271                 continue
1272
1273             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1274                 wrap_in_parentheses(node, child, visible=False)
1275
1276         comma_check = child.type == token.COMMA
1277
1278         check_lpar = isinstance(child, Leaf) and (
1279             child.value in parens_after or comma_check
1280         )
1281
1282
1283 def _normalize_import_from(parent: Node, child: LN, index: int) -> None:
1284     # "import from" nodes store parentheses directly as part of
1285     # the statement
1286     if is_lpar_token(child):
1287         assert is_rpar_token(parent.children[-1])
1288         # make parentheses invisible
1289         child.value = ""
1290         parent.children[-1].value = ""
1291     elif child.type != token.STAR:
1292         # insert invisible parentheses
1293         parent.insert_child(index, Leaf(token.LPAR, ""))
1294         parent.append_child(Leaf(token.RPAR, ""))
1295
1296
1297 def remove_await_parens(node: Node) -> None:
1298     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1299         if (
1300             node.children[1].type == syms.atom
1301             and node.children[1].children[0].type == token.LPAR
1302         ):
1303             if maybe_make_parens_invisible_in_atom(
1304                 node.children[1],
1305                 parent=node,
1306                 remove_brackets_around_comma=True,
1307             ):
1308                 wrap_in_parentheses(node, node.children[1], visible=False)
1309
1310             # Since await is an expression we shouldn't remove
1311             # brackets in cases where this would change
1312             # the AST due to operator precedence.
1313             # Therefore we only aim to remove brackets around
1314             # power nodes that aren't also await expressions themselves.
1315             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1316             # N.B. We've still removed any redundant nested brackets though :)
1317             opening_bracket = cast(Leaf, node.children[1].children[0])
1318             closing_bracket = cast(Leaf, node.children[1].children[-1])
1319             bracket_contents = node.children[1].children[1]
1320             if isinstance(bracket_contents, Node):
1321                 if bracket_contents.type != syms.power:
1322                     ensure_visible(opening_bracket)
1323                     ensure_visible(closing_bracket)
1324                 elif (
1325                     bracket_contents.type == syms.power
1326                     and bracket_contents.children[0].type == token.AWAIT
1327                 ):
1328                     ensure_visible(opening_bracket)
1329                     ensure_visible(closing_bracket)
1330                     # If we are in a nested await then recurse down.
1331                     remove_await_parens(bracket_contents)
1332
1333
1334 def _maybe_wrap_cms_in_parens(
1335     node: Node, mode: Mode, features: Collection[Feature]
1336 ) -> None:
1337     """When enabled and safe, wrap the multiple context managers in invisible parens.
1338
1339     It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.
1340     """
1341     if (
1342         Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features
1343         or Preview.wrap_multiple_context_managers_in_parens not in mode
1344         or len(node.children) <= 2
1345         # If it's an atom, it's already wrapped in parens.
1346         or node.children[1].type == syms.atom
1347     ):
1348         return
1349     colon_index: Optional[int] = None
1350     for i in range(2, len(node.children)):
1351         if node.children[i].type == token.COLON:
1352             colon_index = i
1353             break
1354     if colon_index is not None:
1355         lpar = Leaf(token.LPAR, "")
1356         rpar = Leaf(token.RPAR, "")
1357         context_managers = node.children[1:colon_index]
1358         for child in context_managers:
1359             child.remove()
1360         # After wrapping, the with_stmt will look like this:
1361         #   with_stmt
1362         #     NAME 'with'
1363         #     atom
1364         #       LPAR ''
1365         #       testlist_gexp
1366         #         ... <-- context_managers
1367         #       /testlist_gexp
1368         #       RPAR ''
1369         #     /atom
1370         #     COLON ':'
1371         new_child = Node(
1372             syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]
1373         )
1374         node.insert_child(1, new_child)
1375
1376
1377 def remove_with_parens(node: Node, parent: Node) -> None:
1378     """Recursively hide optional parens in `with` statements."""
1379     # Removing all unnecessary parentheses in with statements in one pass is a tad
1380     # complex as different variations of bracketed statements result in pretty
1381     # different parse trees:
1382     #
1383     # with (open("file")) as f:                       # this is an asexpr_test
1384     #     ...
1385     #
1386     # with (open("file") as f):                       # this is an atom containing an
1387     #     ...                                         # asexpr_test
1388     #
1389     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1390     #     ...                                         # asexpr_test
1391     #
1392     # with (open("file") as f, open("file") as f):    # an atom containing a
1393     #     ...                                         # testlist_gexp which then
1394     #                                                 # contains multiple asexpr_test(s)
1395     if node.type == syms.atom:
1396         if maybe_make_parens_invisible_in_atom(
1397             node,
1398             parent=parent,
1399             remove_brackets_around_comma=True,
1400         ):
1401             wrap_in_parentheses(parent, node, visible=False)
1402         if isinstance(node.children[1], Node):
1403             remove_with_parens(node.children[1], node)
1404     elif node.type == syms.testlist_gexp:
1405         for child in node.children:
1406             if isinstance(child, Node):
1407                 remove_with_parens(child, node)
1408     elif node.type == syms.asexpr_test and not any(
1409         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1410     ):
1411         if maybe_make_parens_invisible_in_atom(
1412             node.children[0],
1413             parent=node,
1414             remove_brackets_around_comma=True,
1415         ):
1416             wrap_in_parentheses(node, node.children[0], visible=False)
1417
1418
1419 def maybe_make_parens_invisible_in_atom(
1420     node: LN,
1421     parent: LN,
1422     remove_brackets_around_comma: bool = False,
1423 ) -> bool:
1424     """If it's safe, make the parens in the atom `node` invisible, recursively.
1425     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1426     as they are redundant.
1427
1428     Returns whether the node should itself be wrapped in invisible parentheses.
1429     """
1430     if (
1431         node.type not in (syms.atom, syms.expr)
1432         or is_empty_tuple(node)
1433         or is_one_tuple(node)
1434         or (is_yield(node) and parent.type != syms.expr_stmt)
1435         or (
1436             # This condition tries to prevent removing non-optional brackets
1437             # around a tuple, however, can be a bit overzealous so we provide
1438             # and option to skip this check for `for` and `with` statements.
1439             not remove_brackets_around_comma
1440             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1441         )
1442         or is_tuple_containing_walrus(node)
1443     ):
1444         return False
1445
1446     if is_walrus_assignment(node):
1447         if parent.type in [
1448             syms.annassign,
1449             syms.expr_stmt,
1450             syms.assert_stmt,
1451             syms.return_stmt,
1452             syms.except_clause,
1453             syms.funcdef,
1454             syms.with_stmt,
1455             syms.tname,
1456             # these ones aren't useful to end users, but they do please fuzzers
1457             syms.for_stmt,
1458             syms.del_stmt,
1459             syms.for_stmt,
1460         ]:
1461             return False
1462
1463     first = node.children[0]
1464     last = node.children[-1]
1465     if is_lpar_token(first) and is_rpar_token(last):
1466         middle = node.children[1]
1467         # make parentheses invisible
1468         if (
1469             # If the prefix of `middle` includes a type comment with
1470             # ignore annotation, then we do not remove the parentheses
1471             not is_type_ignore_comment_string(middle.prefix.strip())
1472         ):
1473             first.value = ""
1474             last.value = ""
1475         maybe_make_parens_invisible_in_atom(
1476             middle,
1477             parent=parent,
1478             remove_brackets_around_comma=remove_brackets_around_comma,
1479         )
1480
1481         if is_atom_with_invisible_parens(middle):
1482             # Strip the invisible parens from `middle` by replacing
1483             # it with the child in-between the invisible parens
1484             middle.replace(middle.children[1])
1485
1486         return False
1487
1488     return True
1489
1490
1491 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1492     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1493
1494     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1495         return False
1496
1497     # We're essentially checking if the body is delimited by commas and there's more
1498     # than one of them (we're excluding the trailing comma and if the delimiter priority
1499     # is still commas, that means there's more).
1500     exclude = set()
1501     trailing_comma = False
1502     try:
1503         last_leaf = line.leaves[-1]
1504         if last_leaf.type == token.COMMA:
1505             trailing_comma = True
1506             exclude.add(id(last_leaf))
1507         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1508     except (IndexError, ValueError):
1509         return False
1510
1511     return max_priority == COMMA_PRIORITY and (
1512         (line.mode.magic_trailing_comma and trailing_comma)
1513         # always explode imports
1514         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1515     )
1516
1517
1518 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1519     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1520
1521     Brackets can be omitted if the entire trailer up to and including
1522     a preceding closing bracket fits in one line.
1523
1524     Yielded sets are cumulative (contain results of previous yields, too).  First
1525     set is empty, unless the line should explode, in which case bracket pairs until
1526     the one that needs to explode are omitted.
1527     """
1528
1529     omit: Set[LeafID] = set()
1530     if not line.magic_trailing_comma:
1531         yield omit
1532
1533     length = 4 * line.depth
1534     opening_bracket: Optional[Leaf] = None
1535     closing_bracket: Optional[Leaf] = None
1536     inner_brackets: Set[LeafID] = set()
1537     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1538         length += leaf_length
1539         if length > line_length:
1540             break
1541
1542         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1543         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1544             break
1545
1546         if opening_bracket:
1547             if leaf is opening_bracket:
1548                 opening_bracket = None
1549             elif leaf.type in CLOSING_BRACKETS:
1550                 prev = line.leaves[index - 1] if index > 0 else None
1551                 if (
1552                     prev
1553                     and prev.type == token.COMMA
1554                     and leaf.opening_bracket is not None
1555                     and not is_one_sequence_between(
1556                         leaf.opening_bracket, leaf, line.leaves
1557                     )
1558                 ):
1559                     # Never omit bracket pairs with trailing commas.
1560                     # We need to explode on those.
1561                     break
1562
1563                 inner_brackets.add(id(leaf))
1564         elif leaf.type in CLOSING_BRACKETS:
1565             prev = line.leaves[index - 1] if index > 0 else None
1566             if prev and prev.type in OPENING_BRACKETS:
1567                 # Empty brackets would fail a split so treat them as "inner"
1568                 # brackets (e.g. only add them to the `omit` set if another
1569                 # pair of brackets was good enough.
1570                 inner_brackets.add(id(leaf))
1571                 continue
1572
1573             if closing_bracket:
1574                 omit.add(id(closing_bracket))
1575                 omit.update(inner_brackets)
1576                 inner_brackets.clear()
1577                 yield omit
1578
1579             if (
1580                 prev
1581                 and prev.type == token.COMMA
1582                 and leaf.opening_bracket is not None
1583                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1584             ):
1585                 # Never omit bracket pairs with trailing commas.
1586                 # We need to explode on those.
1587                 break
1588
1589             if leaf.value:
1590                 opening_bracket = leaf.opening_bracket
1591                 closing_bracket = leaf
1592
1593
1594 def run_transformer(
1595     line: Line,
1596     transform: Transformer,
1597     mode: Mode,
1598     features: Collection[Feature],
1599     *,
1600     line_str: str = "",
1601 ) -> List[Line]:
1602     if not line_str:
1603         line_str = line_to_string(line)
1604     result: List[Line] = []
1605     for transformed_line in transform(line, features, mode):
1606         if str(transformed_line).strip("\n") == line_str:
1607             raise CannotTransform("Line transformer returned an unchanged result")
1608
1609         result.extend(transform_line(transformed_line, mode=mode, features=features))
1610
1611     features_set = set(features)
1612     if (
1613         Feature.FORCE_OPTIONAL_PARENTHESES in features_set
1614         or transform.__class__.__name__ != "rhs"
1615         or not line.bracket_tracker.invisible
1616         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1617         or line.contains_multiline_strings()
1618         or result[0].contains_uncollapsable_type_comments()
1619         or result[0].contains_unsplittable_type_ignore()
1620         or is_line_short_enough(result[0], mode=mode)
1621         # If any leaves have no parents (which _can_ occur since
1622         # `transform(line)` potentially destroys the line's underlying node
1623         # structure), then we can't proceed. Doing so would cause the below
1624         # call to `append_leaves()` to fail.
1625         or any(leaf.parent is None for leaf in line.leaves)
1626     ):
1627         return result
1628
1629     line_copy = line.clone()
1630     append_leaves(line_copy, line, line.leaves)
1631     features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}
1632     second_opinion = run_transformer(
1633         line_copy, transform, mode, features_fop, line_str=line_str
1634     )
1635     if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):
1636         result = second_opinion
1637     return result