src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 from functools import partial, wraps
   5 import sys
   6 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   7
   8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
   9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
  10 from black.nodes import Visitor, syms, is_arith_like, ensure_visible
  11 from black.nodes import (
  12     is_docstring,
  13     is_empty_tuple,
  14     is_one_tuple,
  15     is_one_sequence_between,
  16 )
  17 from black.nodes import is_name_token, is_lpar_token, is_rpar_token
  18 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
  19 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
  20 from black.nodes import wrap_in_parentheses
  21 from black.brackets import max_delimiter_priority_in_atom
  22 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
  23 from black.lines import Line, line_to_string, is_line_short_enough
  24 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
  25 from black.comments import generate_comments, list_comments, FMT_OFF
  26 from black.numerics import normalize_numeric_literal
  27 from black.strings import get_string_prefix, fix_docstring
  28 from black.strings import normalize_string_prefix, normalize_string_quotes
  29 from black.trans import Transformer, CannotTransform, StringMerger, StringSplitter
  30 from black.trans import StringParenWrapper, StringParenStripper, hug_power_op
  31 from black.mode import Mode, Feature, Preview
  32
  33 from blib2to3.pytree import Node, Leaf
  34 from blib2to3.pgen2 import token
  35
  36
  37 # types
  38 LeafID = int
  39 LN = Union[Leaf, Node]
  40
  41
  42 class CannotSplit(CannotTransform):
  43     """A readable split that fits the allotted line length is impossible."""
  44
  45
  46 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  47 # See also https://github.com/mypyc/mypyc/issues/827.
  48 class LineGenerator(Visitor[Line]):
  49     """Generates reformatted Line objects.  Empty lines are not emitted.
  50
  51     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  52     in ways that will no longer stringify to valid Python code on the tree.
  53     """
  54
  55     def __init__(self, mode: Mode) -> None:
  56         self.mode = mode
  57         self.current_line: Line
  58         self.__post_init__()
  59
  60     def line(self, indent: int = 0) -> Iterator[Line]:
  61         """Generate a line.
  62
  63         If the line is empty, only emit if it makes sense.
  64         If the line is too long, split it first and then generate.
  65
  66         If any lines were generated, set up a new current_line.
  67         """
  68         if not self.current_line:
  69             self.current_line.depth += indent
  70             return  # Line is empty, don't emit. Creating a new one unnecessary.
  71
  72         complete_line = self.current_line
  73         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
  74         yield complete_line
  75
  76     def visit_default(self, node: LN) -> Iterator[Line]:
  77         """Default `visit_*()` implementation. Recurses to children of `node`."""
  78         if isinstance(node, Leaf):
  79             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
  80             for comment in generate_comments(node, preview=self.mode.preview):
  81                 if any_open_brackets:
  82                     # any comment within brackets is subject to splitting
  83                     self.current_line.append(comment)
  84                 elif comment.type == token.COMMENT:
  85                     # regular trailing comment
  86                     self.current_line.append(comment)
  87                     yield from self.line()
  88
  89                 else:
  90                     # regular standalone comment
  91                     yield from self.line()
  92
  93                     self.current_line.append(comment)
  94                     yield from self.line()
  95
  96             normalize_prefix(node, inside_brackets=any_open_brackets)
  97             if self.mode.string_normalization and node.type == token.STRING:
  98                 node.value = normalize_string_prefix(node.value)
  99                 node.value = normalize_string_quotes(node.value)
 100             if node.type == token.NUMBER:
 101                 normalize_numeric_literal(node)
 102             if node.type not in WHITESPACE:
 103                 self.current_line.append(node)
 104         yield from super().visit_default(node)
 105
 106     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 107         """Increase indentation level, maybe yield a line."""
 108         # In blib2to3 INDENT never holds comments.
 109         yield from self.line(+1)
 110         yield from self.visit_default(node)
 111
 112     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 113         """Decrease indentation level, maybe yield a line."""
 114         # The current line might still wait for trailing comments.  At DEDENT time
 115         # there won't be any (they would be prefixes on the preceding NEWLINE).
 116         # Emit the line then.
 117         yield from self.line()
 118
 119         # While DEDENT has no value, its prefix may contain standalone comments
 120         # that belong to the current indentation level.  Get 'em.
 121         yield from self.visit_default(node)
 122
 123         # Finally, emit the dedent.
 124         yield from self.line(-1)
 125
 126     def visit_stmt(
 127         self, node: Node, keywords: Set[str], parens: Set[str]
 128     ) -> Iterator[Line]:
 129         """Visit a statement.
 130
 131         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 132         `def`, `with`, `class`, `assert`, and assignments.
 133
 134         The relevant Python language `keywords` for a given statement will be
 135         NAME leaves within it. This methods puts those on a separate line.
 136
 137         `parens` holds a set of string leaf values immediately after which
 138         invisible parens should be put.
 139         """
 140         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
 141         for child in node.children:
 142             if is_name_token(child) and child.value in keywords:
 143                 yield from self.line()
 144
 145             yield from self.visit(child)
 146
 147     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 148         """Visit function definition."""
 149         if Preview.annotation_parens not in self.mode:
 150             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
 151         else:
 152             yield from self.line()
 153
 154             # Remove redundant brackets around return type annotation.
 155             is_return_annotation = False
 156             for child in node.children:
 157                 if child.type == token.RARROW:
 158                     is_return_annotation = True
 159                 elif is_return_annotation:
 160                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 161                         if maybe_make_parens_invisible_in_atom(
 162                             child,
 163                             parent=node,
 164                             remove_brackets_around_comma=False,
 165                         ):
 166                             wrap_in_parentheses(node, child, visible=False)
 167                     else:
 168                         wrap_in_parentheses(node, child, visible=False)
 169                     is_return_annotation = False
 170
 171             for child in node.children:
 172                 yield from self.visit(child)
 173
 174     def visit_match_case(self, node: Node) -> Iterator[Line]:
 175         """Visit either a match or case statement."""
 176         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
 177
 178         yield from self.line()
 179         for child in node.children:
 180             yield from self.visit(child)
 181
 182     def visit_suite(self, node: Node) -> Iterator[Line]:
 183         """Visit a suite."""
 184         if self.mode.is_pyi and is_stub_suite(node):
 185             yield from self.visit(node.children[2])
 186         else:
 187             yield from self.visit_default(node)
 188
 189     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 190         """Visit a statement without nested statements."""
 191         prev_type: Optional[int] = None
 192         for child in node.children:
 193             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 194                 wrap_in_parentheses(node, child, visible=False)
 195             prev_type = child.type
 196
 197         is_suite_like = node.parent and node.parent.type in STATEMENT
 198         if is_suite_like:
 199             if self.mode.is_pyi and is_stub_body(node):
 200                 yield from self.visit_default(node)
 201             else:
 202                 yield from self.line(+1)
 203                 yield from self.visit_default(node)
 204                 yield from self.line(-1)
 205
 206         else:
 207             if (
 208                 not self.mode.is_pyi
 209                 or not node.parent
 210                 or not is_stub_suite(node.parent)
 211             ):
 212                 yield from self.line()
 213             yield from self.visit_default(node)
 214
 215     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 216         """Visit `async def`, `async for`, `async with`."""
 217         yield from self.line()
 218
 219         children = iter(node.children)
 220         for child in children:
 221             yield from self.visit(child)
 222
 223             if child.type == token.ASYNC:
 224                 break
 225
 226         internal_stmt = next(children)
 227         for child in internal_stmt.children:
 228             yield from self.visit(child)
 229
 230     def visit_decorators(self, node: Node) -> Iterator[Line]:
 231         """Visit decorators."""
 232         for child in node.children:
 233             yield from self.line()
 234             yield from self.visit(child)
 235
 236     def visit_power(self, node: Node) -> Iterator[Line]:
 237         for idx, leaf in enumerate(node.children[:-1]):
 238             next_leaf = node.children[idx + 1]
 239
 240             if not isinstance(leaf, Leaf):
 241                 continue
 242
 243             value = leaf.value.lower()
 244             if (
 245                 leaf.type == token.NUMBER
 246                 and next_leaf.type == syms.trailer
 247                 # Ensure that we are in an attribute trailer
 248                 and next_leaf.children[0].type == token.DOT
 249                 # It shouldn't wrap hexadecimal, binary and octal literals
 250                 and not value.startswith(("0x", "0b", "0o"))
 251                 # It shouldn't wrap complex literals
 252                 and "j" not in value
 253             ):
 254                 wrap_in_parentheses(node, leaf)
 255
 256         if Preview.remove_redundant_parens in self.mode:
 257             remove_await_parens(node)
 258
 259         yield from self.visit_default(node)
 260
 261     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 262         """Remove a semicolon and put the other statement on a separate line."""
 263         yield from self.line()
 264
 265     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 266         """End of file. Process outstanding comments and end with a newline."""
 267         yield from self.visit_default(leaf)
 268         yield from self.line()
 269
 270     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 271         if not self.current_line.bracket_tracker.any_open_brackets():
 272             yield from self.line()
 273         yield from self.visit_default(leaf)
 274
 275     def visit_factor(self, node: Node) -> Iterator[Line]:
 276         """Force parentheses between a unary op and a binary power:
 277
 278         -2 ** 8 -> -(2 ** 8)
 279         """
 280         _operator, operand = node.children
 281         if (
 282             operand.type == syms.power
 283             and len(operand.children) == 3
 284             and operand.children[1].type == token.DOUBLESTAR
 285         ):
 286             lpar = Leaf(token.LPAR, "(")
 287             rpar = Leaf(token.RPAR, ")")
 288             index = operand.remove() or 0
 289             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 290         yield from self.visit_default(node)
 291
 292     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 293         if is_docstring(leaf) and "\\\n" not in leaf.value:
 294             # We're ignoring docstrings with backslash newline escapes because changing
 295             # indentation of those changes the AST representation of the code.
 296             docstring = normalize_string_prefix(leaf.value)
 297             prefix = get_string_prefix(docstring)
 298             docstring = docstring[len(prefix) :]  # Remove the prefix
 299             quote_char = docstring[0]
 300             # A natural way to remove the outer quotes is to do:
 301             #   docstring = docstring.strip(quote_char)
 302             # but that breaks on """""x""" (which is '""x').
 303             # So we actually need to remove the first character and the next two
 304             # characters but only if they are the same as the first.
 305             quote_len = 1 if docstring[1] != quote_char else 3
 306             docstring = docstring[quote_len:-quote_len]
 307             docstring_started_empty = not docstring
 308
 309             if is_multiline_string(leaf):
 310                 indent = " " * 4 * self.current_line.depth
 311                 docstring = fix_docstring(docstring, indent)
 312             else:
 313                 docstring = docstring.strip()
 314
 315             if docstring:
 316                 # Add some padding if the docstring starts / ends with a quote mark.
 317                 if docstring[0] == quote_char:
 318                     docstring = " " + docstring
 319                 if docstring[-1] == quote_char:
 320                     docstring += " "
 321                 if docstring[-1] == "\\":
 322                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 323                     if backslash_count % 2:
 324                         # Odd number of tailing backslashes, add some padding to
 325                         # avoid escaping the closing string quote.
 326                         docstring += " "
 327             elif not docstring_started_empty:
 328                 docstring = " "
 329
 330             # We could enforce triple quotes at this point.
 331             quote = quote_char * quote_len
 332             leaf.value = prefix + quote + docstring + quote
 333
 334         yield from self.visit_default(leaf)
 335
 336     def __post_init__(self) -> None:
 337         """You are in a twisty little maze of passages."""
 338         self.current_line = Line(mode=self.mode)
 339
 340         v = self.visit_stmt
 341         Ø: Set[str] = set()
 342         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 343         self.visit_if_stmt = partial(
 344             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 345         )
 346         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 347         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 348         self.visit_try_stmt = partial(
 349             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 350         )
 351         if self.mode.preview:
 352             self.visit_except_clause = partial(
 353                 v, keywords={"except"}, parens={"except"}
 354             )
 355             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 356         else:
 357             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
 358             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
 359         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 360         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 361         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 362         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 363         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 364         self.visit_async_funcdef = self.visit_async_stmt
 365         self.visit_decorated = self.visit_decorators
 366
 367         # PEP 634
 368         self.visit_match_stmt = self.visit_match_case
 369         self.visit_case_block = self.visit_match_case
 370
 371
 372 def transform_line(
 373     line: Line, mode: Mode, features: Collection[Feature] = ()
 374 ) -> Iterator[Line]:
 375     """Transform a `line`, potentially splitting it into many lines.
 376
 377     They should fit in the allotted `line_length` but might not be able to.
 378
 379     `features` are syntactical features that may be used in the output.
 380     """
 381     if line.is_comment:
 382         yield line
 383         return
 384
 385     line_str = line_to_string(line)
 386
 387     ll = mode.line_length
 388     sn = mode.string_normalization
 389     string_merge = StringMerger(ll, sn)
 390     string_paren_strip = StringParenStripper(ll, sn)
 391     string_split = StringSplitter(ll, sn)
 392     string_paren_wrap = StringParenWrapper(ll, sn)
 393
 394     transformers: List[Transformer]
 395     if (
 396         not line.contains_uncollapsable_type_comments()
 397         and not line.should_split_rhs
 398         and not line.magic_trailing_comma
 399         and (
 400             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
 401             or line.contains_unsplittable_type_ignore()
 402         )
 403         and not (line.inside_brackets and line.contains_standalone_comments())
 404     ):
 405         # Only apply basic string preprocessing, since lines shouldn't be split here.
 406         if Preview.string_processing in mode:
 407             transformers = [string_merge, string_paren_strip]
 408         else:
 409             transformers = []
 410     elif line.is_def:
 411         transformers = [left_hand_split]
 412     else:
 413
 414         def _rhs(
 415             self: object, line: Line, features: Collection[Feature]
 416         ) -> Iterator[Line]:
 417             """Wraps calls to `right_hand_split`.
 418
 419             The calls increasingly `omit` right-hand trailers (bracket pairs with
 420             content), meaning the trailers get glued together to split on another
 421             bracket pair instead.
 422             """
 423             for omit in generate_trailers_to_omit(line, mode.line_length):
 424                 lines = list(
 425                     right_hand_split(line, mode.line_length, features, omit=omit)
 426                 )
 427                 # Note: this check is only able to figure out if the first line of the
 428                 # *current* transformation fits in the line length.  This is true only
 429                 # for simple cases.  All others require running more transforms via
 430                 # `transform_line()`.  This check doesn't know if those would succeed.
 431                 if is_line_short_enough(lines[0], line_length=mode.line_length):
 432                     yield from lines
 433                     return
 434
 435             # All splits failed, best effort split with no omits.
 436             # This mostly happens to multiline strings that are by definition
 437             # reported as not fitting a single line, as well as lines that contain
 438             # trailing commas (those have to be exploded).
 439             yield from right_hand_split(
 440                 line, line_length=mode.line_length, features=features
 441             )
 442
 443         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 444         # __name__ attribute which is needed in `run_transformer` further down.
 445         # Unfortunately a nested class breaks mypyc too. So a class must be created
 446         # via type ... https://github.com/mypyc/mypyc/issues/884
 447         rhs = type("rhs", (), {"__call__": _rhs})()
 448
 449         if Preview.string_processing in mode:
 450             if line.inside_brackets:
 451                 transformers = [
 452                     string_merge,
 453                     string_paren_strip,
 454                     string_split,
 455                     delimiter_split,
 456                     standalone_comment_split,
 457                     string_paren_wrap,
 458                     rhs,
 459                 ]
 460             else:
 461                 transformers = [
 462                     string_merge,
 463                     string_paren_strip,
 464                     string_split,
 465                     string_paren_wrap,
 466                     rhs,
 467                 ]
 468         else:
 469             if line.inside_brackets:
 470                 transformers = [delimiter_split, standalone_comment_split, rhs]
 471             else:
 472                 transformers = [rhs]
 473     # It's always safe to attempt hugging of power operations and pretty much every line
 474     # could match.
 475     transformers.append(hug_power_op)
 476
 477     for transform in transformers:
 478         # We are accumulating lines in `result` because we might want to abort
 479         # mission and return the original line in the end, or attempt a different
 480         # split altogether.
 481         try:
 482             result = run_transformer(line, transform, mode, features, line_str=line_str)
 483         except CannotTransform:
 484             continue
 485         else:
 486             yield from result
 487             break
 488
 489     else:
 490         yield line
 491
 492
 493 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
 494     """Split line into many lines, starting with the first matching bracket pair.
 495
 496     Note: this usually looks weird, only use this for function definitions.
 497     Prefer RHS otherwise.  This is why this function is not symmetrical with
 498     :func:`right_hand_split` which also handles optional parentheses.
 499     """
 500     tail_leaves: List[Leaf] = []
 501     body_leaves: List[Leaf] = []
 502     head_leaves: List[Leaf] = []
 503     current_leaves = head_leaves
 504     matching_bracket: Optional[Leaf] = None
 505     for leaf in line.leaves:
 506         if (
 507             current_leaves is body_leaves
 508             and leaf.type in CLOSING_BRACKETS
 509             and leaf.opening_bracket is matching_bracket
 510             and isinstance(matching_bracket, Leaf)
 511         ):
 512             ensure_visible(leaf)
 513             ensure_visible(matching_bracket)
 514             current_leaves = tail_leaves if body_leaves else head_leaves
 515         current_leaves.append(leaf)
 516         if current_leaves is head_leaves:
 517             if leaf.type in OPENING_BRACKETS:
 518                 matching_bracket = leaf
 519                 current_leaves = body_leaves
 520     if not matching_bracket:
 521         raise CannotSplit("No brackets found")
 522
 523     head = bracket_split_build_line(head_leaves, line, matching_bracket)
 524     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
 525     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
 526     bracket_split_succeeded_or_raise(head, body, tail)
 527     for result in (head, body, tail):
 528         if result:
 529             yield result
 530
 531
 532 def right_hand_split(
 533     line: Line,
 534     line_length: int,
 535     features: Collection[Feature] = (),
 536     omit: Collection[LeafID] = (),
 537 ) -> Iterator[Line]:
 538     """Split line into many lines, starting with the last matching bracket pair.
 539
 540     If the split was by optional parentheses, attempt splitting without them, too.
 541     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 542     this split.
 543
 544     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 545     """
 546     tail_leaves: List[Leaf] = []
 547     body_leaves: List[Leaf] = []
 548     head_leaves: List[Leaf] = []
 549     current_leaves = tail_leaves
 550     opening_bracket: Optional[Leaf] = None
 551     closing_bracket: Optional[Leaf] = None
 552     for leaf in reversed(line.leaves):
 553         if current_leaves is body_leaves:
 554             if leaf is opening_bracket:
 555                 current_leaves = head_leaves if body_leaves else tail_leaves
 556         current_leaves.append(leaf)
 557         if current_leaves is tail_leaves:
 558             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 559                 opening_bracket = leaf.opening_bracket
 560                 closing_bracket = leaf
 561                 current_leaves = body_leaves
 562     if not (opening_bracket and closing_bracket and head_leaves):
 563         # If there is no opening or closing_bracket that means the split failed and
 564         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 565         # the matching `opening_bracket` wasn't available on `line` anymore.
 566         raise CannotSplit("No brackets found")
 567
 568     tail_leaves.reverse()
 569     body_leaves.reverse()
 570     head_leaves.reverse()
 571     head = bracket_split_build_line(head_leaves, line, opening_bracket)
 572     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
 573     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
 574     bracket_split_succeeded_or_raise(head, body, tail)
 575     if (
 576         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 577         # the opening bracket is an optional paren
 578         and opening_bracket.type == token.LPAR
 579         and not opening_bracket.value
 580         # the closing bracket is an optional paren
 581         and closing_bracket.type == token.RPAR
 582         and not closing_bracket.value
 583         # it's not an import (optional parens are the only thing we can split on
 584         # in this case; attempting a split without them is a waste of time)
 585         and not line.is_import
 586         # there are no standalone comments in the body
 587         and not body.contains_standalone_comments(0)
 588         # and we can actually remove the parens
 589         and can_omit_invisible_parens(body, line_length)
 590     ):
 591         omit = {id(closing_bracket), *omit}
 592         try:
 593             yield from right_hand_split(line, line_length, features=features, omit=omit)
 594             return
 595
 596         except CannotSplit as e:
 597             if not (
 598                 can_be_split(body)
 599                 or is_line_short_enough(body, line_length=line_length)
 600             ):
 601                 raise CannotSplit(
 602                     "Splitting failed, body is still too long and can't be split."
 603                 ) from e
 604
 605             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
 606                 raise CannotSplit(
 607                     "The current optional pair of parentheses is bound to fail to"
 608                     " satisfy the splitting algorithm because the head or the tail"
 609                     " contains multiline strings which by definition never fit one"
 610                     " line."
 611                 ) from e
 612
 613     ensure_visible(opening_bracket)
 614     ensure_visible(closing_bracket)
 615     for result in (head, body, tail):
 616         if result:
 617             yield result
 618
 619
 620 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 621     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 622
 623     Do nothing otherwise.
 624
 625     A left- or right-hand split is based on a pair of brackets. Content before
 626     (and including) the opening bracket is left on one line, content inside the
 627     brackets is put on a separate line, and finally content starting with and
 628     following the closing bracket is put on a separate line.
 629
 630     Those are called `head`, `body`, and `tail`, respectively. If the split
 631     produced the same line (all content in `head`) or ended up with an empty `body`
 632     and the `tail` is just the closing bracket, then it's considered failed.
 633     """
 634     tail_len = len(str(tail).strip())
 635     if not body:
 636         if tail_len == 0:
 637             raise CannotSplit("Splitting brackets produced the same line")
 638
 639         elif tail_len < 3:
 640             raise CannotSplit(
 641                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 642                 " not worth it"
 643             )
 644
 645
 646 def bracket_split_build_line(
 647     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
 648 ) -> Line:
 649     """Return a new line with given `leaves` and respective comments from `original`.
 650
 651     If `is_body` is True, the result line is one-indented inside brackets and as such
 652     has its first leaf's prefix normalized and a trailing comma added when expected.
 653     """
 654     result = Line(mode=original.mode, depth=original.depth)
 655     if is_body:
 656         result.inside_brackets = True
 657         result.depth += 1
 658         if leaves:
 659             # Since body is a new indent level, remove spurious leading whitespace.
 660             normalize_prefix(leaves[0], inside_brackets=True)
 661             # Ensure a trailing comma for imports and standalone function arguments, but
 662             # be careful not to add one after any comments or within type annotations.
 663             no_commas = (
 664                 original.is_def
 665                 and opening_bracket.value == "("
 666                 and not any(leaf.type == token.COMMA for leaf in leaves)
 667                 # In particular, don't add one within a parenthesized return annotation.
 668                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 669                 # be defined directly in the parent node, the parent of the parent ...
 670                 # and so on depending on how complex the return annotation is.
 671                 # This isn't perfect and there's some false negatives but they are in
 672                 # contexts were a comma is actually fine.
 673                 and not any(
 674                     node.prev_sibling.type == RARROW
 675                     for node in (
 676                         leaves[0].parent,
 677                         getattr(leaves[0].parent, "parent", None),
 678                     )
 679                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 680                 )
 681             )
 682
 683             if original.is_import or no_commas:
 684                 for i in range(len(leaves) - 1, -1, -1):
 685                     if leaves[i].type == STANDALONE_COMMENT:
 686                         continue
 687
 688                     if leaves[i].type != token.COMMA:
 689                         new_comma = Leaf(token.COMMA, ",")
 690                         leaves.insert(i + 1, new_comma)
 691                     break
 692
 693     # Populate the line
 694     for leaf in leaves:
 695         result.append(leaf, preformatted=True)
 696         for comment_after in original.comments_after(leaf):
 697             result.append(comment_after, preformatted=True)
 698     if is_body and should_split_line(result, opening_bracket):
 699         result.should_split_rhs = True
 700     return result
 701
 702
 703 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 704     """Normalize prefix of the first leaf in every line returned by `split_func`.
 705
 706     This is a decorator over relevant split functions.
 707     """
 708
 709     @wraps(split_func)
 710     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 711         for split_line in split_func(line, features):
 712             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 713             yield split_line
 714
 715     return split_wrapper
 716
 717
 718 @dont_increase_indentation
 719 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 720     """Split according to delimiters of the highest priority.
 721
 722     If the appropriate Features are given, the split will add trailing commas
 723     also in function signatures and calls that contain `*` and `**`.
 724     """
 725     try:
 726         last_leaf = line.leaves[-1]
 727     except IndexError:
 728         raise CannotSplit("Line empty") from None
 729
 730     bt = line.bracket_tracker
 731     try:
 732         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 733     except ValueError:
 734         raise CannotSplit("No delimiters found") from None
 735
 736     if delimiter_priority == DOT_PRIORITY:
 737         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 738             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 739
 740     current_line = Line(
 741         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 742     )
 743     lowest_depth = sys.maxsize
 744     trailing_comma_safe = True
 745
 746     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 747         """Append `leaf` to current line or to new line if appending impossible."""
 748         nonlocal current_line
 749         try:
 750             current_line.append_safe(leaf, preformatted=True)
 751         except ValueError:
 752             yield current_line
 753
 754             current_line = Line(
 755                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 756             )
 757             current_line.append(leaf)
 758
 759     for leaf in line.leaves:
 760         yield from append_to_line(leaf)
 761
 762         for comment_after in line.comments_after(leaf):
 763             yield from append_to_line(comment_after)
 764
 765         lowest_depth = min(lowest_depth, leaf.bracket_depth)
 766         if leaf.bracket_depth == lowest_depth:
 767             if is_vararg(leaf, within={syms.typedargslist}):
 768                 trailing_comma_safe = (
 769                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
 770                 )
 771             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
 772                 trailing_comma_safe = (
 773                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
 774                 )
 775
 776         leaf_priority = bt.delimiters.get(id(leaf))
 777         if leaf_priority == delimiter_priority:
 778             yield current_line
 779
 780             current_line = Line(
 781                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 782             )
 783     if current_line:
 784         if (
 785             trailing_comma_safe
 786             and delimiter_priority == COMMA_PRIORITY
 787             and current_line.leaves[-1].type != token.COMMA
 788             and current_line.leaves[-1].type != STANDALONE_COMMENT
 789         ):
 790             new_comma = Leaf(token.COMMA, ",")
 791             current_line.append(new_comma)
 792         yield current_line
 793
 794
 795 @dont_increase_indentation
 796 def standalone_comment_split(
 797     line: Line, features: Collection[Feature] = ()
 798 ) -> Iterator[Line]:
 799     """Split standalone comments from the rest of the line."""
 800     if not line.contains_standalone_comments(0):
 801         raise CannotSplit("Line does not have any standalone comments")
 802
 803     current_line = Line(
 804         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 805     )
 806
 807     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 808         """Append `leaf` to current line or to new line if appending impossible."""
 809         nonlocal current_line
 810         try:
 811             current_line.append_safe(leaf, preformatted=True)
 812         except ValueError:
 813             yield current_line
 814
 815             current_line = Line(
 816                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 817             )
 818             current_line.append(leaf)
 819
 820     for leaf in line.leaves:
 821         yield from append_to_line(leaf)
 822
 823         for comment_after in line.comments_after(leaf):
 824             yield from append_to_line(comment_after)
 825
 826     if current_line:
 827         yield current_line
 828
 829
 830 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
 831     """Leave existing extra newlines if not `inside_brackets`. Remove everything
 832     else.
 833
 834     Note: don't use backslashes for formatting or you'll lose your voting rights.
 835     """
 836     if not inside_brackets:
 837         spl = leaf.prefix.split("#")
 838         if "\\" not in spl[0]:
 839             nl_count = spl[-1].count("\n")
 840             if len(spl) > 1:
 841                 nl_count -= 1
 842             leaf.prefix = "\n" * nl_count
 843             return
 844
 845     leaf.prefix = ""
 846
 847
 848 def normalize_invisible_parens(
 849     node: Node, parens_after: Set[str], *, preview: bool
 850 ) -> None:
 851     """Make existing optional parentheses invisible or create new ones.
 852
 853     `parens_after` is a set of string leaf values immediately after which parens
 854     should be put.
 855
 856     Standardizes on visible parentheses for single-element tuples, and keeps
 857     existing visible parentheses for other tuples and generator expressions.
 858     """
 859     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
 860         if pc.value in FMT_OFF:
 861             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
 862             return
 863     check_lpar = False
 864     for index, child in enumerate(list(node.children)):
 865         # Fixes a bug where invisible parens are not properly stripped from
 866         # assignment statements that contain type annotations.
 867         if isinstance(child, Node) and child.type == syms.annassign:
 868             normalize_invisible_parens(
 869                 child, parens_after=parens_after, preview=preview
 870             )
 871
 872         # Add parentheses around long tuple unpacking in assignments.
 873         if (
 874             index == 0
 875             and isinstance(child, Node)
 876             and child.type == syms.testlist_star_expr
 877         ):
 878             check_lpar = True
 879
 880         if check_lpar:
 881             if (
 882                 preview
 883                 and child.type == syms.atom
 884                 and node.type == syms.for_stmt
 885                 and isinstance(child.prev_sibling, Leaf)
 886                 and child.prev_sibling.type == token.NAME
 887                 and child.prev_sibling.value == "for"
 888             ):
 889                 if maybe_make_parens_invisible_in_atom(
 890                     child,
 891                     parent=node,
 892                     remove_brackets_around_comma=True,
 893                 ):
 894                     wrap_in_parentheses(node, child, visible=False)
 895             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
 896                 remove_with_parens(child, node)
 897             elif child.type == syms.atom:
 898                 if maybe_make_parens_invisible_in_atom(
 899                     child,
 900                     parent=node,
 901                 ):
 902                     wrap_in_parentheses(node, child, visible=False)
 903             elif is_one_tuple(child):
 904                 wrap_in_parentheses(node, child, visible=True)
 905             elif node.type == syms.import_from:
 906                 # "import from" nodes store parentheses directly as part of
 907                 # the statement
 908                 if is_lpar_token(child):
 909                     assert is_rpar_token(node.children[-1])
 910                     # make parentheses invisible
 911                     child.value = ""
 912                     node.children[-1].value = ""
 913                 elif child.type != token.STAR:
 914                     # insert invisible parentheses
 915                     node.insert_child(index, Leaf(token.LPAR, ""))
 916                     node.append_child(Leaf(token.RPAR, ""))
 917                 break
 918             elif (
 919                 index == 1
 920                 and child.type == token.STAR
 921                 and node.type == syms.except_clause
 922             ):
 923                 # In except* (PEP 654), the star is actually part of
 924                 # of the keyword. So we need to skip the insertion of
 925                 # invisible parentheses to work more precisely.
 926                 continue
 927
 928             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
 929                 wrap_in_parentheses(node, child, visible=False)
 930
 931         comma_check = child.type == token.COMMA if preview else False
 932
 933         check_lpar = isinstance(child, Leaf) and (
 934             child.value in parens_after or comma_check
 935         )
 936
 937
 938 def remove_await_parens(node: Node) -> None:
 939     if node.children[0].type == token.AWAIT and len(node.children) > 1:
 940         if (
 941             node.children[1].type == syms.atom
 942             and node.children[1].children[0].type == token.LPAR
 943         ):
 944             if maybe_make_parens_invisible_in_atom(
 945                 node.children[1],
 946                 parent=node,
 947                 remove_brackets_around_comma=True,
 948             ):
 949                 wrap_in_parentheses(node, node.children[1], visible=False)
 950
 951             # Since await is an expression we shouldn't remove
 952             # brackets in cases where this would change
 953             # the AST due to operator precedence.
 954             # Therefore we only aim to remove brackets around
 955             # power nodes that aren't also await expressions themselves.
 956             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
 957             # N.B. We've still removed any redundant nested brackets though :)
 958             opening_bracket = cast(Leaf, node.children[1].children[0])
 959             closing_bracket = cast(Leaf, node.children[1].children[-1])
 960             bracket_contents = cast(Node, node.children[1].children[1])
 961             if bracket_contents.type != syms.power:
 962                 ensure_visible(opening_bracket)
 963                 ensure_visible(closing_bracket)
 964             elif (
 965                 bracket_contents.type == syms.power
 966                 and bracket_contents.children[0].type == token.AWAIT
 967             ):
 968                 ensure_visible(opening_bracket)
 969                 ensure_visible(closing_bracket)
 970                 # If we are in a nested await then recurse down.
 971                 remove_await_parens(bracket_contents)
 972
 973
 974 def remove_with_parens(node: Node, parent: Node) -> None:
 975     """Recursively hide optional parens in `with` statements."""
 976     # Removing all unnecessary parentheses in with statements in one pass is a tad
 977     # complex as different variations of bracketed statements result in pretty
 978     # different parse trees:
 979     #
 980     # with (open("file")) as f:                       # this is an asexpr_test
 981     #     ...
 982     #
 983     # with (open("file") as f):                       # this is an atom containing an
 984     #     ...                                         # asexpr_test
 985     #
 986     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
 987     #     ...                                         # asexpr_test
 988     #
 989     # with (open("file") as f, open("file") as f):    # an atom containing a
 990     #     ...                                         # testlist_gexp which then
 991     #                                                 # contains multiple asexpr_test(s)
 992     if node.type == syms.atom:
 993         if maybe_make_parens_invisible_in_atom(
 994             node,
 995             parent=parent,
 996             remove_brackets_around_comma=True,
 997         ):
 998             wrap_in_parentheses(parent, node, visible=False)
 999         if isinstance(node.children[1], Node):
1000             remove_with_parens(node.children[1], node)
1001     elif node.type == syms.testlist_gexp:
1002         for child in node.children:
1003             if isinstance(child, Node):
1004                 remove_with_parens(child, node)
1005     elif node.type == syms.asexpr_test and not any(
1006         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1007     ):
1008         if maybe_make_parens_invisible_in_atom(
1009             node.children[0],
1010             parent=node,
1011             remove_brackets_around_comma=True,
1012         ):
1013             wrap_in_parentheses(node, node.children[0], visible=False)
1014
1015
1016 def maybe_make_parens_invisible_in_atom(
1017     node: LN,
1018     parent: LN,
1019     remove_brackets_around_comma: bool = False,
1020 ) -> bool:
1021     """If it's safe, make the parens in the atom `node` invisible, recursively.
1022     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1023     as they are redundant.
1024
1025     Returns whether the node should itself be wrapped in invisible parentheses.
1026     """
1027     if (
1028         node.type != syms.atom
1029         or is_empty_tuple(node)
1030         or is_one_tuple(node)
1031         or (is_yield(node) and parent.type != syms.expr_stmt)
1032         or (
1033             # This condition tries to prevent removing non-optional brackets
1034             # around a tuple, however, can be a bit overzealous so we provide
1035             # and option to skip this check for `for` and `with` statements.
1036             not remove_brackets_around_comma
1037             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1038         )
1039     ):
1040         return False
1041
1042     if is_walrus_assignment(node):
1043         if parent.type in [
1044             syms.annassign,
1045             syms.expr_stmt,
1046             syms.assert_stmt,
1047             syms.return_stmt,
1048             # these ones aren't useful to end users, but they do please fuzzers
1049             syms.for_stmt,
1050             syms.del_stmt,
1051         ]:
1052             return False
1053
1054     first = node.children[0]
1055     last = node.children[-1]
1056     if is_lpar_token(first) and is_rpar_token(last):
1057         middle = node.children[1]
1058         # make parentheses invisible
1059         first.value = ""
1060         last.value = ""
1061         maybe_make_parens_invisible_in_atom(
1062             middle,
1063             parent=parent,
1064             remove_brackets_around_comma=remove_brackets_around_comma,
1065         )
1066
1067         if is_atom_with_invisible_parens(middle):
1068             # Strip the invisible parens from `middle` by replacing
1069             # it with the child in-between the invisible parens
1070             middle.replace(middle.children[1])
1071
1072         return False
1073
1074     return True
1075
1076
1077 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1078     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1079
1080     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1081         return False
1082
1083     # We're essentially checking if the body is delimited by commas and there's more
1084     # than one of them (we're excluding the trailing comma and if the delimiter priority
1085     # is still commas, that means there's more).
1086     exclude = set()
1087     trailing_comma = False
1088     try:
1089         last_leaf = line.leaves[-1]
1090         if last_leaf.type == token.COMMA:
1091             trailing_comma = True
1092             exclude.add(id(last_leaf))
1093         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1094     except (IndexError, ValueError):
1095         return False
1096
1097     return max_priority == COMMA_PRIORITY and (
1098         (line.mode.magic_trailing_comma and trailing_comma)
1099         # always explode imports
1100         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1101     )
1102
1103
1104 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1105     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1106
1107     Brackets can be omitted if the entire trailer up to and including
1108     a preceding closing bracket fits in one line.
1109
1110     Yielded sets are cumulative (contain results of previous yields, too).  First
1111     set is empty, unless the line should explode, in which case bracket pairs until
1112     the one that needs to explode are omitted.
1113     """
1114
1115     omit: Set[LeafID] = set()
1116     if not line.magic_trailing_comma:
1117         yield omit
1118
1119     length = 4 * line.depth
1120     opening_bracket: Optional[Leaf] = None
1121     closing_bracket: Optional[Leaf] = None
1122     inner_brackets: Set[LeafID] = set()
1123     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1124         length += leaf_length
1125         if length > line_length:
1126             break
1127
1128         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1129         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1130             break
1131
1132         if opening_bracket:
1133             if leaf is opening_bracket:
1134                 opening_bracket = None
1135             elif leaf.type in CLOSING_BRACKETS:
1136                 prev = line.leaves[index - 1] if index > 0 else None
1137                 if (
1138                     prev
1139                     and prev.type == token.COMMA
1140                     and leaf.opening_bracket is not None
1141                     and not is_one_sequence_between(
1142                         leaf.opening_bracket, leaf, line.leaves
1143                     )
1144                 ):
1145                     # Never omit bracket pairs with trailing commas.
1146                     # We need to explode on those.
1147                     break
1148
1149                 inner_brackets.add(id(leaf))
1150         elif leaf.type in CLOSING_BRACKETS:
1151             prev = line.leaves[index - 1] if index > 0 else None
1152             if prev and prev.type in OPENING_BRACKETS:
1153                 # Empty brackets would fail a split so treat them as "inner"
1154                 # brackets (e.g. only add them to the `omit` set if another
1155                 # pair of brackets was good enough.
1156                 inner_brackets.add(id(leaf))
1157                 continue
1158
1159             if closing_bracket:
1160                 omit.add(id(closing_bracket))
1161                 omit.update(inner_brackets)
1162                 inner_brackets.clear()
1163                 yield omit
1164
1165             if (
1166                 prev
1167                 and prev.type == token.COMMA
1168                 and leaf.opening_bracket is not None
1169                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1170             ):
1171                 # Never omit bracket pairs with trailing commas.
1172                 # We need to explode on those.
1173                 break
1174
1175             if leaf.value:
1176                 opening_bracket = leaf.opening_bracket
1177                 closing_bracket = leaf
1178
1179
1180 def run_transformer(
1181     line: Line,
1182     transform: Transformer,
1183     mode: Mode,
1184     features: Collection[Feature],
1185     *,
1186     line_str: str = "",
1187 ) -> List[Line]:
1188     if not line_str:
1189         line_str = line_to_string(line)
1190     result: List[Line] = []
1191     for transformed_line in transform(line, features):
1192         if str(transformed_line).strip("\n") == line_str:
1193             raise CannotTransform("Line transformer returned an unchanged result")
1194
1195         result.extend(transform_line(transformed_line, mode=mode, features=features))
1196
1197     if (
1198         transform.__class__.__name__ != "rhs"
1199         or not line.bracket_tracker.invisible
1200         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1201         or line.contains_multiline_strings()
1202         or result[0].contains_uncollapsable_type_comments()
1203         or result[0].contains_unsplittable_type_ignore()
1204         or is_line_short_enough(result[0], line_length=mode.line_length)
1205         # If any leaves have no parents (which _can_ occur since
1206         # `transform(line)` potentially destroys the line's underlying node
1207         # structure), then we can't proceed. Doing so would cause the below
1208         # call to `append_leaves()` to fail.
1209         or any(leaf.parent is None for leaf in line.leaves)
1210     ):
1211         return result
1212
1213     line_copy = line.clone()
1214     append_leaves(line_copy, line, line.leaves)
1215     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1216     second_opinion = run_transformer(
1217         line_copy, transform, mode, features_fop, line_str=line_str
1218     )
1219     if all(
1220         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1221     ):
1222         result = second_opinion
1223     return result