src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 from functools import partial, wraps
   5 import sys
   6 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   7
   8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
   9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
  10 from black.nodes import Visitor, syms, is_arith_like, ensure_visible
  11 from black.nodes import (
  12     is_docstring,
  13     is_empty_tuple,
  14     is_one_tuple,
  15     is_one_sequence_between,
  16 )
  17 from black.nodes import is_name_token, is_lpar_token, is_rpar_token
  18 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
  19 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
  20 from black.nodes import wrap_in_parentheses
  21 from black.brackets import max_delimiter_priority_in_atom
  22 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
  23 from black.lines import Line, line_to_string, is_line_short_enough
  24 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
  25 from black.comments import generate_comments, list_comments, FMT_OFF
  26 from black.numerics import normalize_numeric_literal
  27 from black.strings import get_string_prefix, fix_docstring
  28 from black.strings import normalize_string_prefix, normalize_string_quotes
  29 from black.trans import Transformer, CannotTransform, StringMerger, StringSplitter
  30 from black.trans import StringParenWrapper, StringParenStripper, hug_power_op
  31 from black.mode import Mode, Feature, Preview
  32
  33 from blib2to3.pytree import Node, Leaf
  34 from blib2to3.pgen2 import token
  35
  36
  37 # types
  38 LeafID = int
  39 LN = Union[Leaf, Node]
  40
  41
  42 class CannotSplit(CannotTransform):
  43     """A readable split that fits the allotted line length is impossible."""
  44
  45
  46 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  47 # See also https://github.com/mypyc/mypyc/issues/827.
  48 class LineGenerator(Visitor[Line]):
  49     """Generates reformatted Line objects.  Empty lines are not emitted.
  50
  51     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  52     in ways that will no longer stringify to valid Python code on the tree.
  53     """
  54
  55     def __init__(self, mode: Mode) -> None:
  56         self.mode = mode
  57         self.current_line: Line
  58         self.__post_init__()
  59
  60     def line(self, indent: int = 0) -> Iterator[Line]:
  61         """Generate a line.
  62
  63         If the line is empty, only emit if it makes sense.
  64         If the line is too long, split it first and then generate.
  65
  66         If any lines were generated, set up a new current_line.
  67         """
  68         if not self.current_line:
  69             self.current_line.depth += indent
  70             return  # Line is empty, don't emit. Creating a new one unnecessary.
  71
  72         complete_line = self.current_line
  73         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
  74         yield complete_line
  75
  76     def visit_default(self, node: LN) -> Iterator[Line]:
  77         """Default `visit_*()` implementation. Recurses to children of `node`."""
  78         if isinstance(node, Leaf):
  79             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
  80             for comment in generate_comments(node, preview=self.mode.preview):
  81                 if any_open_brackets:
  82                     # any comment within brackets is subject to splitting
  83                     self.current_line.append(comment)
  84                 elif comment.type == token.COMMENT:
  85                     # regular trailing comment
  86                     self.current_line.append(comment)
  87                     yield from self.line()
  88
  89                 else:
  90                     # regular standalone comment
  91                     yield from self.line()
  92
  93                     self.current_line.append(comment)
  94                     yield from self.line()
  95
  96             normalize_prefix(node, inside_brackets=any_open_brackets)
  97             if self.mode.string_normalization and node.type == token.STRING:
  98                 node.value = normalize_string_prefix(node.value)
  99                 node.value = normalize_string_quotes(node.value)
 100             if node.type == token.NUMBER:
 101                 normalize_numeric_literal(node)
 102             if node.type not in WHITESPACE:
 103                 self.current_line.append(node)
 104         yield from super().visit_default(node)
 105
 106     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 107         """Increase indentation level, maybe yield a line."""
 108         # In blib2to3 INDENT never holds comments.
 109         yield from self.line(+1)
 110         yield from self.visit_default(node)
 111
 112     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 113         """Decrease indentation level, maybe yield a line."""
 114         # The current line might still wait for trailing comments.  At DEDENT time
 115         # there won't be any (they would be prefixes on the preceding NEWLINE).
 116         # Emit the line then.
 117         yield from self.line()
 118
 119         # While DEDENT has no value, its prefix may contain standalone comments
 120         # that belong to the current indentation level.  Get 'em.
 121         yield from self.visit_default(node)
 122
 123         # Finally, emit the dedent.
 124         yield from self.line(-1)
 125
 126     def visit_stmt(
 127         self, node: Node, keywords: Set[str], parens: Set[str]
 128     ) -> Iterator[Line]:
 129         """Visit a statement.
 130
 131         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 132         `def`, `with`, `class`, `assert`, and assignments.
 133
 134         The relevant Python language `keywords` for a given statement will be
 135         NAME leaves within it. This methods puts those on a separate line.
 136
 137         `parens` holds a set of string leaf values immediately after which
 138         invisible parens should be put.
 139         """
 140         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
 141         for child in node.children:
 142             if is_name_token(child) and child.value in keywords:
 143                 yield from self.line()
 144
 145             yield from self.visit(child)
 146
 147     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 148         """Visit function definition."""
 149         if Preview.annotation_parens not in self.mode:
 150             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
 151         else:
 152             yield from self.line()
 153
 154             # Remove redundant brackets around return type annotation.
 155             is_return_annotation = False
 156             for child in node.children:
 157                 if child.type == token.RARROW:
 158                     is_return_annotation = True
 159                 elif is_return_annotation:
 160                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 161                         if maybe_make_parens_invisible_in_atom(
 162                             child,
 163                             parent=node,
 164                             remove_brackets_around_comma=False,
 165                         ):
 166                             wrap_in_parentheses(node, child, visible=False)
 167                     else:
 168                         wrap_in_parentheses(node, child, visible=False)
 169                     is_return_annotation = False
 170
 171             for child in node.children:
 172                 yield from self.visit(child)
 173
 174     def visit_match_case(self, node: Node) -> Iterator[Line]:
 175         """Visit either a match or case statement."""
 176         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
 177
 178         yield from self.line()
 179         for child in node.children:
 180             yield from self.visit(child)
 181
 182     def visit_suite(self, node: Node) -> Iterator[Line]:
 183         """Visit a suite."""
 184         if self.mode.is_pyi and is_stub_suite(node):
 185             yield from self.visit(node.children[2])
 186         else:
 187             yield from self.visit_default(node)
 188
 189     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 190         """Visit a statement without nested statements."""
 191         prev_type: Optional[int] = None
 192         for child in node.children:
 193             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 194                 wrap_in_parentheses(node, child, visible=False)
 195             prev_type = child.type
 196
 197         is_suite_like = node.parent and node.parent.type in STATEMENT
 198         if is_suite_like:
 199             if self.mode.is_pyi and is_stub_body(node):
 200                 yield from self.visit_default(node)
 201             else:
 202                 yield from self.line(+1)
 203                 yield from self.visit_default(node)
 204                 yield from self.line(-1)
 205
 206         else:
 207             if (
 208                 not self.mode.is_pyi
 209                 or not node.parent
 210                 or not is_stub_suite(node.parent)
 211             ):
 212                 yield from self.line()
 213             yield from self.visit_default(node)
 214
 215     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 216         """Visit `async def`, `async for`, `async with`."""
 217         yield from self.line()
 218
 219         children = iter(node.children)
 220         for child in children:
 221             yield from self.visit(child)
 222
 223             if child.type == token.ASYNC:
 224                 break
 225
 226         internal_stmt = next(children)
 227         for child in internal_stmt.children:
 228             yield from self.visit(child)
 229
 230     def visit_decorators(self, node: Node) -> Iterator[Line]:
 231         """Visit decorators."""
 232         for child in node.children:
 233             yield from self.line()
 234             yield from self.visit(child)
 235
 236     def visit_power(self, node: Node) -> Iterator[Line]:
 237         for idx, leaf in enumerate(node.children[:-1]):
 238             next_leaf = node.children[idx + 1]
 239
 240             if not isinstance(leaf, Leaf):
 241                 continue
 242
 243             value = leaf.value.lower()
 244             if (
 245                 leaf.type == token.NUMBER
 246                 and next_leaf.type == syms.trailer
 247                 # Ensure that we are in an attribute trailer
 248                 and next_leaf.children[0].type == token.DOT
 249                 # It shouldn't wrap hexadecimal, binary and octal literals
 250                 and not value.startswith(("0x", "0b", "0o"))
 251                 # It shouldn't wrap complex literals
 252                 and "j" not in value
 253             ):
 254                 wrap_in_parentheses(node, leaf)
 255
 256         if Preview.remove_redundant_parens in self.mode:
 257             remove_await_parens(node)
 258
 259         yield from self.visit_default(node)
 260
 261     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 262         """Remove a semicolon and put the other statement on a separate line."""
 263         yield from self.line()
 264
 265     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 266         """End of file. Process outstanding comments and end with a newline."""
 267         yield from self.visit_default(leaf)
 268         yield from self.line()
 269
 270     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 271         if not self.current_line.bracket_tracker.any_open_brackets():
 272             yield from self.line()
 273         yield from self.visit_default(leaf)
 274
 275     def visit_factor(self, node: Node) -> Iterator[Line]:
 276         """Force parentheses between a unary op and a binary power:
 277
 278         -2 ** 8 -> -(2 ** 8)
 279         """
 280         _operator, operand = node.children
 281         if (
 282             operand.type == syms.power
 283             and len(operand.children) == 3
 284             and operand.children[1].type == token.DOUBLESTAR
 285         ):
 286             lpar = Leaf(token.LPAR, "(")
 287             rpar = Leaf(token.RPAR, ")")
 288             index = operand.remove() or 0
 289             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 290         yield from self.visit_default(node)
 291
 292     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 293         if is_docstring(leaf) and "\\\n" not in leaf.value:
 294             # We're ignoring docstrings with backslash newline escapes because changing
 295             # indentation of those changes the AST representation of the code.
 296             docstring = normalize_string_prefix(leaf.value)
 297             prefix = get_string_prefix(docstring)
 298             docstring = docstring[len(prefix) :]  # Remove the prefix
 299             quote_char = docstring[0]
 300             # A natural way to remove the outer quotes is to do:
 301             #   docstring = docstring.strip(quote_char)
 302             # but that breaks on """""x""" (which is '""x').
 303             # So we actually need to remove the first character and the next two
 304             # characters but only if they are the same as the first.
 305             quote_len = 1 if docstring[1] != quote_char else 3
 306             docstring = docstring[quote_len:-quote_len]
 307             docstring_started_empty = not docstring
 308             indent = " " * 4 * self.current_line.depth
 309
 310             if is_multiline_string(leaf):
 311                 docstring = fix_docstring(docstring, indent)
 312             else:
 313                 docstring = docstring.strip()
 314
 315             if docstring:
 316                 # Add some padding if the docstring starts / ends with a quote mark.
 317                 if docstring[0] == quote_char:
 318                     docstring = " " + docstring
 319                 if docstring[-1] == quote_char:
 320                     docstring += " "
 321                 if docstring[-1] == "\\":
 322                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 323                     if backslash_count % 2:
 324                         # Odd number of tailing backslashes, add some padding to
 325                         # avoid escaping the closing string quote.
 326                         docstring += " "
 327             elif not docstring_started_empty:
 328                 docstring = " "
 329
 330             # We could enforce triple quotes at this point.
 331             quote = quote_char * quote_len
 332
 333             if Preview.long_docstring_quotes_on_newline in self.mode:
 334                 # We need to find the length of the last line of the docstring
 335                 # to find if we can add the closing quotes to the line without
 336                 # exceeding the maximum line length.
 337                 # If docstring is one line, then we need to add the length
 338                 # of the indent, prefix, and starting quotes. Ending quote are
 339                 # handled later
 340                 lines = docstring.splitlines()
 341                 last_line_length = len(lines[-1]) if docstring else 0
 342
 343                 if len(lines) == 1:
 344                     last_line_length += len(indent) + len(prefix) + quote_len
 345
 346                 # If adding closing quotes would cause the last line to exceed
 347                 # the maximum line length then put a line break before the
 348                 # closing quotes
 349                 if last_line_length + quote_len > self.mode.line_length:
 350                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 351                 else:
 352                     leaf.value = prefix + quote + docstring + quote
 353             else:
 354                 leaf.value = prefix + quote + docstring + quote
 355
 356         yield from self.visit_default(leaf)
 357
 358     def __post_init__(self) -> None:
 359         """You are in a twisty little maze of passages."""
 360         self.current_line = Line(mode=self.mode)
 361
 362         v = self.visit_stmt
 363         Ø: Set[str] = set()
 364         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 365         self.visit_if_stmt = partial(
 366             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 367         )
 368         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 369         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 370         self.visit_try_stmt = partial(
 371             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 372         )
 373         if self.mode.preview:
 374             self.visit_except_clause = partial(
 375                 v, keywords={"except"}, parens={"except"}
 376             )
 377             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 378         else:
 379             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
 380             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
 381         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 382         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 383         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 384         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 385         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 386         self.visit_async_funcdef = self.visit_async_stmt
 387         self.visit_decorated = self.visit_decorators
 388
 389         # PEP 634
 390         self.visit_match_stmt = self.visit_match_case
 391         self.visit_case_block = self.visit_match_case
 392
 393
 394 def transform_line(
 395     line: Line, mode: Mode, features: Collection[Feature] = ()
 396 ) -> Iterator[Line]:
 397     """Transform a `line`, potentially splitting it into many lines.
 398
 399     They should fit in the allotted `line_length` but might not be able to.
 400
 401     `features` are syntactical features that may be used in the output.
 402     """
 403     if line.is_comment:
 404         yield line
 405         return
 406
 407     line_str = line_to_string(line)
 408
 409     ll = mode.line_length
 410     sn = mode.string_normalization
 411     string_merge = StringMerger(ll, sn)
 412     string_paren_strip = StringParenStripper(ll, sn)
 413     string_split = StringSplitter(ll, sn)
 414     string_paren_wrap = StringParenWrapper(ll, sn)
 415
 416     transformers: List[Transformer]
 417     if (
 418         not line.contains_uncollapsable_type_comments()
 419         and not line.should_split_rhs
 420         and not line.magic_trailing_comma
 421         and (
 422             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
 423             or line.contains_unsplittable_type_ignore()
 424         )
 425         and not (line.inside_brackets and line.contains_standalone_comments())
 426     ):
 427         # Only apply basic string preprocessing, since lines shouldn't be split here.
 428         if Preview.string_processing in mode:
 429             transformers = [string_merge, string_paren_strip]
 430         else:
 431             transformers = []
 432     elif line.is_def:
 433         transformers = [left_hand_split]
 434     else:
 435
 436         def _rhs(
 437             self: object, line: Line, features: Collection[Feature]
 438         ) -> Iterator[Line]:
 439             """Wraps calls to `right_hand_split`.
 440
 441             The calls increasingly `omit` right-hand trailers (bracket pairs with
 442             content), meaning the trailers get glued together to split on another
 443             bracket pair instead.
 444             """
 445             for omit in generate_trailers_to_omit(line, mode.line_length):
 446                 lines = list(
 447                     right_hand_split(line, mode.line_length, features, omit=omit)
 448                 )
 449                 # Note: this check is only able to figure out if the first line of the
 450                 # *current* transformation fits in the line length.  This is true only
 451                 # for simple cases.  All others require running more transforms via
 452                 # `transform_line()`.  This check doesn't know if those would succeed.
 453                 if is_line_short_enough(lines[0], line_length=mode.line_length):
 454                     yield from lines
 455                     return
 456
 457             # All splits failed, best effort split with no omits.
 458             # This mostly happens to multiline strings that are by definition
 459             # reported as not fitting a single line, as well as lines that contain
 460             # trailing commas (those have to be exploded).
 461             yield from right_hand_split(
 462                 line, line_length=mode.line_length, features=features
 463             )
 464
 465         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 466         # __name__ attribute which is needed in `run_transformer` further down.
 467         # Unfortunately a nested class breaks mypyc too. So a class must be created
 468         # via type ... https://github.com/mypyc/mypyc/issues/884
 469         rhs = type("rhs", (), {"__call__": _rhs})()
 470
 471         if Preview.string_processing in mode:
 472             if line.inside_brackets:
 473                 transformers = [
 474                     string_merge,
 475                     string_paren_strip,
 476                     string_split,
 477                     delimiter_split,
 478                     standalone_comment_split,
 479                     string_paren_wrap,
 480                     rhs,
 481                 ]
 482             else:
 483                 transformers = [
 484                     string_merge,
 485                     string_paren_strip,
 486                     string_split,
 487                     string_paren_wrap,
 488                     rhs,
 489                 ]
 490         else:
 491             if line.inside_brackets:
 492                 transformers = [delimiter_split, standalone_comment_split, rhs]
 493             else:
 494                 transformers = [rhs]
 495     # It's always safe to attempt hugging of power operations and pretty much every line
 496     # could match.
 497     transformers.append(hug_power_op)
 498
 499     for transform in transformers:
 500         # We are accumulating lines in `result` because we might want to abort
 501         # mission and return the original line in the end, or attempt a different
 502         # split altogether.
 503         try:
 504             result = run_transformer(line, transform, mode, features, line_str=line_str)
 505         except CannotTransform:
 506             continue
 507         else:
 508             yield from result
 509             break
 510
 511     else:
 512         yield line
 513
 514
 515 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
 516     """Split line into many lines, starting with the first matching bracket pair.
 517
 518     Note: this usually looks weird, only use this for function definitions.
 519     Prefer RHS otherwise.  This is why this function is not symmetrical with
 520     :func:`right_hand_split` which also handles optional parentheses.
 521     """
 522     tail_leaves: List[Leaf] = []
 523     body_leaves: List[Leaf] = []
 524     head_leaves: List[Leaf] = []
 525     current_leaves = head_leaves
 526     matching_bracket: Optional[Leaf] = None
 527     for leaf in line.leaves:
 528         if (
 529             current_leaves is body_leaves
 530             and leaf.type in CLOSING_BRACKETS
 531             and leaf.opening_bracket is matching_bracket
 532             and isinstance(matching_bracket, Leaf)
 533         ):
 534             ensure_visible(leaf)
 535             ensure_visible(matching_bracket)
 536             current_leaves = tail_leaves if body_leaves else head_leaves
 537         current_leaves.append(leaf)
 538         if current_leaves is head_leaves:
 539             if leaf.type in OPENING_BRACKETS:
 540                 matching_bracket = leaf
 541                 current_leaves = body_leaves
 542     if not matching_bracket:
 543         raise CannotSplit("No brackets found")
 544
 545     head = bracket_split_build_line(head_leaves, line, matching_bracket)
 546     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
 547     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
 548     bracket_split_succeeded_or_raise(head, body, tail)
 549     for result in (head, body, tail):
 550         if result:
 551             yield result
 552
 553
 554 def right_hand_split(
 555     line: Line,
 556     line_length: int,
 557     features: Collection[Feature] = (),
 558     omit: Collection[LeafID] = (),
 559 ) -> Iterator[Line]:
 560     """Split line into many lines, starting with the last matching bracket pair.
 561
 562     If the split was by optional parentheses, attempt splitting without them, too.
 563     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 564     this split.
 565
 566     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 567     """
 568     tail_leaves: List[Leaf] = []
 569     body_leaves: List[Leaf] = []
 570     head_leaves: List[Leaf] = []
 571     current_leaves = tail_leaves
 572     opening_bracket: Optional[Leaf] = None
 573     closing_bracket: Optional[Leaf] = None
 574     for leaf in reversed(line.leaves):
 575         if current_leaves is body_leaves:
 576             if leaf is opening_bracket:
 577                 current_leaves = head_leaves if body_leaves else tail_leaves
 578         current_leaves.append(leaf)
 579         if current_leaves is tail_leaves:
 580             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 581                 opening_bracket = leaf.opening_bracket
 582                 closing_bracket = leaf
 583                 current_leaves = body_leaves
 584     if not (opening_bracket and closing_bracket and head_leaves):
 585         # If there is no opening or closing_bracket that means the split failed and
 586         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 587         # the matching `opening_bracket` wasn't available on `line` anymore.
 588         raise CannotSplit("No brackets found")
 589
 590     tail_leaves.reverse()
 591     body_leaves.reverse()
 592     head_leaves.reverse()
 593     head = bracket_split_build_line(head_leaves, line, opening_bracket)
 594     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
 595     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
 596     bracket_split_succeeded_or_raise(head, body, tail)
 597     if (
 598         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 599         # the opening bracket is an optional paren
 600         and opening_bracket.type == token.LPAR
 601         and not opening_bracket.value
 602         # the closing bracket is an optional paren
 603         and closing_bracket.type == token.RPAR
 604         and not closing_bracket.value
 605         # it's not an import (optional parens are the only thing we can split on
 606         # in this case; attempting a split without them is a waste of time)
 607         and not line.is_import
 608         # there are no standalone comments in the body
 609         and not body.contains_standalone_comments(0)
 610         # and we can actually remove the parens
 611         and can_omit_invisible_parens(body, line_length)
 612     ):
 613         omit = {id(closing_bracket), *omit}
 614         try:
 615             yield from right_hand_split(line, line_length, features=features, omit=omit)
 616             return
 617
 618         except CannotSplit as e:
 619             if not (
 620                 can_be_split(body)
 621                 or is_line_short_enough(body, line_length=line_length)
 622             ):
 623                 raise CannotSplit(
 624                     "Splitting failed, body is still too long and can't be split."
 625                 ) from e
 626
 627             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
 628                 raise CannotSplit(
 629                     "The current optional pair of parentheses is bound to fail to"
 630                     " satisfy the splitting algorithm because the head or the tail"
 631                     " contains multiline strings which by definition never fit one"
 632                     " line."
 633                 ) from e
 634
 635     ensure_visible(opening_bracket)
 636     ensure_visible(closing_bracket)
 637     for result in (head, body, tail):
 638         if result:
 639             yield result
 640
 641
 642 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 643     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 644
 645     Do nothing otherwise.
 646
 647     A left- or right-hand split is based on a pair of brackets. Content before
 648     (and including) the opening bracket is left on one line, content inside the
 649     brackets is put on a separate line, and finally content starting with and
 650     following the closing bracket is put on a separate line.
 651
 652     Those are called `head`, `body`, and `tail`, respectively. If the split
 653     produced the same line (all content in `head`) or ended up with an empty `body`
 654     and the `tail` is just the closing bracket, then it's considered failed.
 655     """
 656     tail_len = len(str(tail).strip())
 657     if not body:
 658         if tail_len == 0:
 659             raise CannotSplit("Splitting brackets produced the same line")
 660
 661         elif tail_len < 3:
 662             raise CannotSplit(
 663                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 664                 " not worth it"
 665             )
 666
 667
 668 def bracket_split_build_line(
 669     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
 670 ) -> Line:
 671     """Return a new line with given `leaves` and respective comments from `original`.
 672
 673     If `is_body` is True, the result line is one-indented inside brackets and as such
 674     has its first leaf's prefix normalized and a trailing comma added when expected.
 675     """
 676     result = Line(mode=original.mode, depth=original.depth)
 677     if is_body:
 678         result.inside_brackets = True
 679         result.depth += 1
 680         if leaves:
 681             # Since body is a new indent level, remove spurious leading whitespace.
 682             normalize_prefix(leaves[0], inside_brackets=True)
 683             # Ensure a trailing comma for imports and standalone function arguments, but
 684             # be careful not to add one after any comments or within type annotations.
 685             no_commas = (
 686                 original.is_def
 687                 and opening_bracket.value == "("
 688                 and not any(leaf.type == token.COMMA for leaf in leaves)
 689                 # In particular, don't add one within a parenthesized return annotation.
 690                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 691                 # be defined directly in the parent node, the parent of the parent ...
 692                 # and so on depending on how complex the return annotation is.
 693                 # This isn't perfect and there's some false negatives but they are in
 694                 # contexts were a comma is actually fine.
 695                 and not any(
 696                     node.prev_sibling.type == RARROW
 697                     for node in (
 698                         leaves[0].parent,
 699                         getattr(leaves[0].parent, "parent", None),
 700                     )
 701                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 702                 )
 703             )
 704
 705             if original.is_import or no_commas:
 706                 for i in range(len(leaves) - 1, -1, -1):
 707                     if leaves[i].type == STANDALONE_COMMENT:
 708                         continue
 709
 710                     if leaves[i].type != token.COMMA:
 711                         new_comma = Leaf(token.COMMA, ",")
 712                         leaves.insert(i + 1, new_comma)
 713                     break
 714
 715     # Populate the line
 716     for leaf in leaves:
 717         result.append(leaf, preformatted=True)
 718         for comment_after in original.comments_after(leaf):
 719             result.append(comment_after, preformatted=True)
 720     if is_body and should_split_line(result, opening_bracket):
 721         result.should_split_rhs = True
 722     return result
 723
 724
 725 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 726     """Normalize prefix of the first leaf in every line returned by `split_func`.
 727
 728     This is a decorator over relevant split functions.
 729     """
 730
 731     @wraps(split_func)
 732     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 733         for split_line in split_func(line, features):
 734             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 735             yield split_line
 736
 737     return split_wrapper
 738
 739
 740 @dont_increase_indentation
 741 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 742     """Split according to delimiters of the highest priority.
 743
 744     If the appropriate Features are given, the split will add trailing commas
 745     also in function signatures and calls that contain `*` and `**`.
 746     """
 747     try:
 748         last_leaf = line.leaves[-1]
 749     except IndexError:
 750         raise CannotSplit("Line empty") from None
 751
 752     bt = line.bracket_tracker
 753     try:
 754         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 755     except ValueError:
 756         raise CannotSplit("No delimiters found") from None
 757
 758     if delimiter_priority == DOT_PRIORITY:
 759         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 760             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 761
 762     current_line = Line(
 763         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 764     )
 765     lowest_depth = sys.maxsize
 766     trailing_comma_safe = True
 767
 768     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 769         """Append `leaf` to current line or to new line if appending impossible."""
 770         nonlocal current_line
 771         try:
 772             current_line.append_safe(leaf, preformatted=True)
 773         except ValueError:
 774             yield current_line
 775
 776             current_line = Line(
 777                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 778             )
 779             current_line.append(leaf)
 780
 781     for leaf in line.leaves:
 782         yield from append_to_line(leaf)
 783
 784         for comment_after in line.comments_after(leaf):
 785             yield from append_to_line(comment_after)
 786
 787         lowest_depth = min(lowest_depth, leaf.bracket_depth)
 788         if leaf.bracket_depth == lowest_depth:
 789             if is_vararg(leaf, within={syms.typedargslist}):
 790                 trailing_comma_safe = (
 791                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
 792                 )
 793             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
 794                 trailing_comma_safe = (
 795                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
 796                 )
 797
 798         leaf_priority = bt.delimiters.get(id(leaf))
 799         if leaf_priority == delimiter_priority:
 800             yield current_line
 801
 802             current_line = Line(
 803                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 804             )
 805     if current_line:
 806         if (
 807             trailing_comma_safe
 808             and delimiter_priority == COMMA_PRIORITY
 809             and current_line.leaves[-1].type != token.COMMA
 810             and current_line.leaves[-1].type != STANDALONE_COMMENT
 811         ):
 812             new_comma = Leaf(token.COMMA, ",")
 813             current_line.append(new_comma)
 814         yield current_line
 815
 816
 817 @dont_increase_indentation
 818 def standalone_comment_split(
 819     line: Line, features: Collection[Feature] = ()
 820 ) -> Iterator[Line]:
 821     """Split standalone comments from the rest of the line."""
 822     if not line.contains_standalone_comments(0):
 823         raise CannotSplit("Line does not have any standalone comments")
 824
 825     current_line = Line(
 826         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 827     )
 828
 829     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 830         """Append `leaf` to current line or to new line if appending impossible."""
 831         nonlocal current_line
 832         try:
 833             current_line.append_safe(leaf, preformatted=True)
 834         except ValueError:
 835             yield current_line
 836
 837             current_line = Line(
 838                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 839             )
 840             current_line.append(leaf)
 841
 842     for leaf in line.leaves:
 843         yield from append_to_line(leaf)
 844
 845         for comment_after in line.comments_after(leaf):
 846             yield from append_to_line(comment_after)
 847
 848     if current_line:
 849         yield current_line
 850
 851
 852 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
 853     """Leave existing extra newlines if not `inside_brackets`. Remove everything
 854     else.
 855
 856     Note: don't use backslashes for formatting or you'll lose your voting rights.
 857     """
 858     if not inside_brackets:
 859         spl = leaf.prefix.split("#")
 860         if "\\" not in spl[0]:
 861             nl_count = spl[-1].count("\n")
 862             if len(spl) > 1:
 863                 nl_count -= 1
 864             leaf.prefix = "\n" * nl_count
 865             return
 866
 867     leaf.prefix = ""
 868
 869
 870 def normalize_invisible_parens(
 871     node: Node, parens_after: Set[str], *, preview: bool
 872 ) -> None:
 873     """Make existing optional parentheses invisible or create new ones.
 874
 875     `parens_after` is a set of string leaf values immediately after which parens
 876     should be put.
 877
 878     Standardizes on visible parentheses for single-element tuples, and keeps
 879     existing visible parentheses for other tuples and generator expressions.
 880     """
 881     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
 882         if pc.value in FMT_OFF:
 883             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
 884             return
 885     check_lpar = False
 886     for index, child in enumerate(list(node.children)):
 887         # Fixes a bug where invisible parens are not properly stripped from
 888         # assignment statements that contain type annotations.
 889         if isinstance(child, Node) and child.type == syms.annassign:
 890             normalize_invisible_parens(
 891                 child, parens_after=parens_after, preview=preview
 892             )
 893
 894         # Add parentheses around long tuple unpacking in assignments.
 895         if (
 896             index == 0
 897             and isinstance(child, Node)
 898             and child.type == syms.testlist_star_expr
 899         ):
 900             check_lpar = True
 901
 902         if check_lpar:
 903             if (
 904                 preview
 905                 and child.type == syms.atom
 906                 and node.type == syms.for_stmt
 907                 and isinstance(child.prev_sibling, Leaf)
 908                 and child.prev_sibling.type == token.NAME
 909                 and child.prev_sibling.value == "for"
 910             ):
 911                 if maybe_make_parens_invisible_in_atom(
 912                     child,
 913                     parent=node,
 914                     remove_brackets_around_comma=True,
 915                 ):
 916                     wrap_in_parentheses(node, child, visible=False)
 917             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
 918                 remove_with_parens(child, node)
 919             elif child.type == syms.atom:
 920                 if maybe_make_parens_invisible_in_atom(
 921                     child,
 922                     parent=node,
 923                 ):
 924                     wrap_in_parentheses(node, child, visible=False)
 925             elif is_one_tuple(child):
 926                 wrap_in_parentheses(node, child, visible=True)
 927             elif node.type == syms.import_from:
 928                 # "import from" nodes store parentheses directly as part of
 929                 # the statement
 930                 if is_lpar_token(child):
 931                     assert is_rpar_token(node.children[-1])
 932                     # make parentheses invisible
 933                     child.value = ""
 934                     node.children[-1].value = ""
 935                 elif child.type != token.STAR:
 936                     # insert invisible parentheses
 937                     node.insert_child(index, Leaf(token.LPAR, ""))
 938                     node.append_child(Leaf(token.RPAR, ""))
 939                 break
 940             elif (
 941                 index == 1
 942                 and child.type == token.STAR
 943                 and node.type == syms.except_clause
 944             ):
 945                 # In except* (PEP 654), the star is actually part of
 946                 # of the keyword. So we need to skip the insertion of
 947                 # invisible parentheses to work more precisely.
 948                 continue
 949
 950             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
 951                 wrap_in_parentheses(node, child, visible=False)
 952
 953         comma_check = child.type == token.COMMA if preview else False
 954
 955         check_lpar = isinstance(child, Leaf) and (
 956             child.value in parens_after or comma_check
 957         )
 958
 959
 960 def remove_await_parens(node: Node) -> None:
 961     if node.children[0].type == token.AWAIT and len(node.children) > 1:
 962         if (
 963             node.children[1].type == syms.atom
 964             and node.children[1].children[0].type == token.LPAR
 965         ):
 966             if maybe_make_parens_invisible_in_atom(
 967                 node.children[1],
 968                 parent=node,
 969                 remove_brackets_around_comma=True,
 970             ):
 971                 wrap_in_parentheses(node, node.children[1], visible=False)
 972
 973             # Since await is an expression we shouldn't remove
 974             # brackets in cases where this would change
 975             # the AST due to operator precedence.
 976             # Therefore we only aim to remove brackets around
 977             # power nodes that aren't also await expressions themselves.
 978             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
 979             # N.B. We've still removed any redundant nested brackets though :)
 980             opening_bracket = cast(Leaf, node.children[1].children[0])
 981             closing_bracket = cast(Leaf, node.children[1].children[-1])
 982             bracket_contents = cast(Node, node.children[1].children[1])
 983             if bracket_contents.type != syms.power:
 984                 ensure_visible(opening_bracket)
 985                 ensure_visible(closing_bracket)
 986             elif (
 987                 bracket_contents.type == syms.power
 988                 and bracket_contents.children[0].type == token.AWAIT
 989             ):
 990                 ensure_visible(opening_bracket)
 991                 ensure_visible(closing_bracket)
 992                 # If we are in a nested await then recurse down.
 993                 remove_await_parens(bracket_contents)
 994
 995
 996 def remove_with_parens(node: Node, parent: Node) -> None:
 997     """Recursively hide optional parens in `with` statements."""
 998     # Removing all unnecessary parentheses in with statements in one pass is a tad
 999     # complex as different variations of bracketed statements result in pretty
1000     # different parse trees:
1001     #
1002     # with (open("file")) as f:                       # this is an asexpr_test
1003     #     ...
1004     #
1005     # with (open("file") as f):                       # this is an atom containing an
1006     #     ...                                         # asexpr_test
1007     #
1008     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1009     #     ...                                         # asexpr_test
1010     #
1011     # with (open("file") as f, open("file") as f):    # an atom containing a
1012     #     ...                                         # testlist_gexp which then
1013     #                                                 # contains multiple asexpr_test(s)
1014     if node.type == syms.atom:
1015         if maybe_make_parens_invisible_in_atom(
1016             node,
1017             parent=parent,
1018             remove_brackets_around_comma=True,
1019         ):
1020             wrap_in_parentheses(parent, node, visible=False)
1021         if isinstance(node.children[1], Node):
1022             remove_with_parens(node.children[1], node)
1023     elif node.type == syms.testlist_gexp:
1024         for child in node.children:
1025             if isinstance(child, Node):
1026                 remove_with_parens(child, node)
1027     elif node.type == syms.asexpr_test and not any(
1028         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1029     ):
1030         if maybe_make_parens_invisible_in_atom(
1031             node.children[0],
1032             parent=node,
1033             remove_brackets_around_comma=True,
1034         ):
1035             wrap_in_parentheses(node, node.children[0], visible=False)
1036
1037
1038 def maybe_make_parens_invisible_in_atom(
1039     node: LN,
1040     parent: LN,
1041     remove_brackets_around_comma: bool = False,
1042 ) -> bool:
1043     """If it's safe, make the parens in the atom `node` invisible, recursively.
1044     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1045     as they are redundant.
1046
1047     Returns whether the node should itself be wrapped in invisible parentheses.
1048     """
1049     if (
1050         node.type != syms.atom
1051         or is_empty_tuple(node)
1052         or is_one_tuple(node)
1053         or (is_yield(node) and parent.type != syms.expr_stmt)
1054         or (
1055             # This condition tries to prevent removing non-optional brackets
1056             # around a tuple, however, can be a bit overzealous so we provide
1057             # and option to skip this check for `for` and `with` statements.
1058             not remove_brackets_around_comma
1059             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1060         )
1061     ):
1062         return False
1063
1064     if is_walrus_assignment(node):
1065         if parent.type in [
1066             syms.annassign,
1067             syms.expr_stmt,
1068             syms.assert_stmt,
1069             syms.return_stmt,
1070             # these ones aren't useful to end users, but they do please fuzzers
1071             syms.for_stmt,
1072             syms.del_stmt,
1073         ]:
1074             return False
1075
1076     first = node.children[0]
1077     last = node.children[-1]
1078     if is_lpar_token(first) and is_rpar_token(last):
1079         middle = node.children[1]
1080         # make parentheses invisible
1081         first.value = ""
1082         last.value = ""
1083         maybe_make_parens_invisible_in_atom(
1084             middle,
1085             parent=parent,
1086             remove_brackets_around_comma=remove_brackets_around_comma,
1087         )
1088
1089         if is_atom_with_invisible_parens(middle):
1090             # Strip the invisible parens from `middle` by replacing
1091             # it with the child in-between the invisible parens
1092             middle.replace(middle.children[1])
1093
1094         return False
1095
1096     return True
1097
1098
1099 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1100     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1101
1102     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1103         return False
1104
1105     # We're essentially checking if the body is delimited by commas and there's more
1106     # than one of them (we're excluding the trailing comma and if the delimiter priority
1107     # is still commas, that means there's more).
1108     exclude = set()
1109     trailing_comma = False
1110     try:
1111         last_leaf = line.leaves[-1]
1112         if last_leaf.type == token.COMMA:
1113             trailing_comma = True
1114             exclude.add(id(last_leaf))
1115         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1116     except (IndexError, ValueError):
1117         return False
1118
1119     return max_priority == COMMA_PRIORITY and (
1120         (line.mode.magic_trailing_comma and trailing_comma)
1121         # always explode imports
1122         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1123     )
1124
1125
1126 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1127     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1128
1129     Brackets can be omitted if the entire trailer up to and including
1130     a preceding closing bracket fits in one line.
1131
1132     Yielded sets are cumulative (contain results of previous yields, too).  First
1133     set is empty, unless the line should explode, in which case bracket pairs until
1134     the one that needs to explode are omitted.
1135     """
1136
1137     omit: Set[LeafID] = set()
1138     if not line.magic_trailing_comma:
1139         yield omit
1140
1141     length = 4 * line.depth
1142     opening_bracket: Optional[Leaf] = None
1143     closing_bracket: Optional[Leaf] = None
1144     inner_brackets: Set[LeafID] = set()
1145     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1146         length += leaf_length
1147         if length > line_length:
1148             break
1149
1150         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1151         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1152             break
1153
1154         if opening_bracket:
1155             if leaf is opening_bracket:
1156                 opening_bracket = None
1157             elif leaf.type in CLOSING_BRACKETS:
1158                 prev = line.leaves[index - 1] if index > 0 else None
1159                 if (
1160                     prev
1161                     and prev.type == token.COMMA
1162                     and leaf.opening_bracket is not None
1163                     and not is_one_sequence_between(
1164                         leaf.opening_bracket, leaf, line.leaves
1165                     )
1166                 ):
1167                     # Never omit bracket pairs with trailing commas.
1168                     # We need to explode on those.
1169                     break
1170
1171                 inner_brackets.add(id(leaf))
1172         elif leaf.type in CLOSING_BRACKETS:
1173             prev = line.leaves[index - 1] if index > 0 else None
1174             if prev and prev.type in OPENING_BRACKETS:
1175                 # Empty brackets would fail a split so treat them as "inner"
1176                 # brackets (e.g. only add them to the `omit` set if another
1177                 # pair of brackets was good enough.
1178                 inner_brackets.add(id(leaf))
1179                 continue
1180
1181             if closing_bracket:
1182                 omit.add(id(closing_bracket))
1183                 omit.update(inner_brackets)
1184                 inner_brackets.clear()
1185                 yield omit
1186
1187             if (
1188                 prev
1189                 and prev.type == token.COMMA
1190                 and leaf.opening_bracket is not None
1191                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1192             ):
1193                 # Never omit bracket pairs with trailing commas.
1194                 # We need to explode on those.
1195                 break
1196
1197             if leaf.value:
1198                 opening_bracket = leaf.opening_bracket
1199                 closing_bracket = leaf
1200
1201
1202 def run_transformer(
1203     line: Line,
1204     transform: Transformer,
1205     mode: Mode,
1206     features: Collection[Feature],
1207     *,
1208     line_str: str = "",
1209 ) -> List[Line]:
1210     if not line_str:
1211         line_str = line_to_string(line)
1212     result: List[Line] = []
1213     for transformed_line in transform(line, features):
1214         if str(transformed_line).strip("\n") == line_str:
1215             raise CannotTransform("Line transformer returned an unchanged result")
1216
1217         result.extend(transform_line(transformed_line, mode=mode, features=features))
1218
1219     if (
1220         transform.__class__.__name__ != "rhs"
1221         or not line.bracket_tracker.invisible
1222         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1223         or line.contains_multiline_strings()
1224         or result[0].contains_uncollapsable_type_comments()
1225         or result[0].contains_unsplittable_type_ignore()
1226         or is_line_short_enough(result[0], line_length=mode.line_length)
1227         # If any leaves have no parents (which _can_ occur since
1228         # `transform(line)` potentially destroys the line's underlying node
1229         # structure), then we can't proceed. Doing so would cause the below
1230         # call to `append_leaves()` to fail.
1231         or any(leaf.parent is None for leaf in line.leaves)
1232     ):
1233         return result
1234
1235     line_copy = line.clone()
1236     append_leaves(line_copy, line, line.leaves)
1237     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1238     second_opinion = run_transformer(
1239         line_copy, transform, mode, features_fop, line_str=line_str
1240     )
1241     if all(
1242         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1243     ):
1244         result = second_opinion
1245     return result