src/black/linegen.py

   1 """
   2 Generating lines of code.
   3 """
   4 from functools import partial, wraps
   5 import sys
   6 from typing import Collection, Iterator, List, Optional, Set, Union, cast
   7
   8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
   9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
  10 from black.nodes import Visitor, syms, is_arith_like, ensure_visible
  11 from black.nodes import (
  12     is_docstring,
  13     is_empty_tuple,
  14     is_one_tuple,
  15     is_one_sequence_between,
  16 )
  17 from black.nodes import is_name_token, is_lpar_token, is_rpar_token
  18 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
  19 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
  20 from black.nodes import wrap_in_parentheses
  21 from black.brackets import max_delimiter_priority_in_atom
  22 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
  23 from black.lines import Line, line_to_string, is_line_short_enough
  24 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
  25 from black.comments import generate_comments, list_comments, FMT_OFF
  26 from black.numerics import normalize_numeric_literal
  27 from black.strings import get_string_prefix, fix_docstring
  28 from black.strings import normalize_string_prefix, normalize_string_quotes
  29 from black.trans import Transformer, CannotTransform, StringMerger, StringSplitter
  30 from black.trans import StringParenWrapper, StringParenStripper, hug_power_op
  31 from black.mode import Mode, Feature, Preview
  32
  33 from blib2to3.pytree import Node, Leaf
  34 from blib2to3.pgen2 import token
  35
  36
  37 # types
  38 LeafID = int
  39 LN = Union[Leaf, Node]
  40
  41
  42 class CannotSplit(CannotTransform):
  43     """A readable split that fits the allotted line length is impossible."""
  44
  45
  46 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
  47 # See also https://github.com/mypyc/mypyc/issues/827.
  48 class LineGenerator(Visitor[Line]):
  49     """Generates reformatted Line objects.  Empty lines are not emitted.
  50
  51     Note: destroys the tree it's visiting by mutating prefixes of its leaves
  52     in ways that will no longer stringify to valid Python code on the tree.
  53     """
  54
  55     def __init__(self, mode: Mode) -> None:
  56         self.mode = mode
  57         self.current_line: Line
  58         self.__post_init__()
  59
  60     def line(self, indent: int = 0) -> Iterator[Line]:
  61         """Generate a line.
  62
  63         If the line is empty, only emit if it makes sense.
  64         If the line is too long, split it first and then generate.
  65
  66         If any lines were generated, set up a new current_line.
  67         """
  68         if not self.current_line:
  69             self.current_line.depth += indent
  70             return  # Line is empty, don't emit. Creating a new one unnecessary.
  71
  72         complete_line = self.current_line
  73         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
  74         yield complete_line
  75
  76     def visit_default(self, node: LN) -> Iterator[Line]:
  77         """Default `visit_*()` implementation. Recurses to children of `node`."""
  78         if isinstance(node, Leaf):
  79             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
  80             for comment in generate_comments(node, preview=self.mode.preview):
  81                 if any_open_brackets:
  82                     # any comment within brackets is subject to splitting
  83                     self.current_line.append(comment)
  84                 elif comment.type == token.COMMENT:
  85                     # regular trailing comment
  86                     self.current_line.append(comment)
  87                     yield from self.line()
  88
  89                 else:
  90                     # regular standalone comment
  91                     yield from self.line()
  92
  93                     self.current_line.append(comment)
  94                     yield from self.line()
  95
  96             normalize_prefix(node, inside_brackets=any_open_brackets)
  97             if self.mode.string_normalization and node.type == token.STRING:
  98                 node.value = normalize_string_prefix(node.value)
  99                 node.value = normalize_string_quotes(node.value)
 100             if node.type == token.NUMBER:
 101                 normalize_numeric_literal(node)
 102             if node.type not in WHITESPACE:
 103                 self.current_line.append(node)
 104         yield from super().visit_default(node)
 105
 106     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
 107         """Increase indentation level, maybe yield a line."""
 108         # In blib2to3 INDENT never holds comments.
 109         yield from self.line(+1)
 110         yield from self.visit_default(node)
 111
 112     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
 113         """Decrease indentation level, maybe yield a line."""
 114         # The current line might still wait for trailing comments.  At DEDENT time
 115         # there won't be any (they would be prefixes on the preceding NEWLINE).
 116         # Emit the line then.
 117         yield from self.line()
 118
 119         # While DEDENT has no value, its prefix may contain standalone comments
 120         # that belong to the current indentation level.  Get 'em.
 121         yield from self.visit_default(node)
 122
 123         # Finally, emit the dedent.
 124         yield from self.line(-1)
 125
 126     def visit_stmt(
 127         self, node: Node, keywords: Set[str], parens: Set[str]
 128     ) -> Iterator[Line]:
 129         """Visit a statement.
 130
 131         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
 132         `def`, `with`, `class`, `assert`, and assignments.
 133
 134         The relevant Python language `keywords` for a given statement will be
 135         NAME leaves within it. This methods puts those on a separate line.
 136
 137         `parens` holds a set of string leaf values immediately after which
 138         invisible parens should be put.
 139         """
 140         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
 141         for child in node.children:
 142             if is_name_token(child) and child.value in keywords:
 143                 yield from self.line()
 144
 145             yield from self.visit(child)
 146
 147     def visit_funcdef(self, node: Node) -> Iterator[Line]:
 148         """Visit function definition."""
 149         if Preview.annotation_parens not in self.mode:
 150             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
 151         else:
 152             yield from self.line()
 153
 154             # Remove redundant brackets around return type annotation.
 155             is_return_annotation = False
 156             for child in node.children:
 157                 if child.type == token.RARROW:
 158                     is_return_annotation = True
 159                 elif is_return_annotation:
 160                     if child.type == syms.atom and child.children[0].type == token.LPAR:
 161                         if maybe_make_parens_invisible_in_atom(
 162                             child,
 163                             parent=node,
 164                             remove_brackets_around_comma=False,
 165                         ):
 166                             wrap_in_parentheses(node, child, visible=False)
 167                     else:
 168                         wrap_in_parentheses(node, child, visible=False)
 169                     is_return_annotation = False
 170
 171             for child in node.children:
 172                 yield from self.visit(child)
 173
 174     def visit_match_case(self, node: Node) -> Iterator[Line]:
 175         """Visit either a match or case statement."""
 176         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
 177
 178         yield from self.line()
 179         for child in node.children:
 180             yield from self.visit(child)
 181
 182     def visit_suite(self, node: Node) -> Iterator[Line]:
 183         """Visit a suite."""
 184         if self.mode.is_pyi and is_stub_suite(node):
 185             yield from self.visit(node.children[2])
 186         else:
 187             yield from self.visit_default(node)
 188
 189     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
 190         """Visit a statement without nested statements."""
 191         prev_type: Optional[int] = None
 192         for child in node.children:
 193             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
 194                 wrap_in_parentheses(node, child, visible=False)
 195             prev_type = child.type
 196
 197         is_suite_like = node.parent and node.parent.type in STATEMENT
 198         if is_suite_like:
 199             if self.mode.is_pyi and is_stub_body(node):
 200                 yield from self.visit_default(node)
 201             else:
 202                 yield from self.line(+1)
 203                 yield from self.visit_default(node)
 204                 yield from self.line(-1)
 205
 206         else:
 207             if (
 208                 not self.mode.is_pyi
 209                 or not node.parent
 210                 or not is_stub_suite(node.parent)
 211             ):
 212                 yield from self.line()
 213             yield from self.visit_default(node)
 214
 215     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
 216         """Visit `async def`, `async for`, `async with`."""
 217         yield from self.line()
 218
 219         children = iter(node.children)
 220         for child in children:
 221             yield from self.visit(child)
 222
 223             if child.type == token.ASYNC:
 224                 break
 225
 226         internal_stmt = next(children)
 227         for child in internal_stmt.children:
 228             yield from self.visit(child)
 229
 230     def visit_decorators(self, node: Node) -> Iterator[Line]:
 231         """Visit decorators."""
 232         for child in node.children:
 233             yield from self.line()
 234             yield from self.visit(child)
 235
 236     def visit_power(self, node: Node) -> Iterator[Line]:
 237         for idx, leaf in enumerate(node.children[:-1]):
 238             next_leaf = node.children[idx + 1]
 239
 240             if not isinstance(leaf, Leaf):
 241                 continue
 242
 243             value = leaf.value.lower()
 244             if (
 245                 leaf.type == token.NUMBER
 246                 and next_leaf.type == syms.trailer
 247                 # Ensure that we are in an attribute trailer
 248                 and next_leaf.children[0].type == token.DOT
 249                 # It shouldn't wrap hexadecimal, binary and octal literals
 250                 and not value.startswith(("0x", "0b", "0o"))
 251                 # It shouldn't wrap complex literals
 252                 and "j" not in value
 253             ):
 254                 wrap_in_parentheses(node, leaf)
 255
 256         if Preview.remove_redundant_parens in self.mode:
 257             remove_await_parens(node)
 258
 259         yield from self.visit_default(node)
 260
 261     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
 262         """Remove a semicolon and put the other statement on a separate line."""
 263         yield from self.line()
 264
 265     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
 266         """End of file. Process outstanding comments and end with a newline."""
 267         yield from self.visit_default(leaf)
 268         yield from self.line()
 269
 270     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
 271         if not self.current_line.bracket_tracker.any_open_brackets():
 272             yield from self.line()
 273         yield from self.visit_default(leaf)
 274
 275     def visit_factor(self, node: Node) -> Iterator[Line]:
 276         """Force parentheses between a unary op and a binary power:
 277
 278         -2 ** 8 -> -(2 ** 8)
 279         """
 280         _operator, operand = node.children
 281         if (
 282             operand.type == syms.power
 283             and len(operand.children) == 3
 284             and operand.children[1].type == token.DOUBLESTAR
 285         ):
 286             lpar = Leaf(token.LPAR, "(")
 287             rpar = Leaf(token.RPAR, ")")
 288             index = operand.remove() or 0
 289             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
 290         yield from self.visit_default(node)
 291
 292     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
 293         if is_docstring(leaf) and "\\\n" not in leaf.value:
 294             # We're ignoring docstrings with backslash newline escapes because changing
 295             # indentation of those changes the AST representation of the code.
 296             docstring = normalize_string_prefix(leaf.value)
 297             prefix = get_string_prefix(docstring)
 298             docstring = docstring[len(prefix) :]  # Remove the prefix
 299             quote_char = docstring[0]
 300             # A natural way to remove the outer quotes is to do:
 301             #   docstring = docstring.strip(quote_char)
 302             # but that breaks on """""x""" (which is '""x').
 303             # So we actually need to remove the first character and the next two
 304             # characters but only if they are the same as the first.
 305             quote_len = 1 if docstring[1] != quote_char else 3
 306             docstring = docstring[quote_len:-quote_len]
 307             docstring_started_empty = not docstring
 308             indent = " " * 4 * self.current_line.depth
 309
 310             if is_multiline_string(leaf):
 311                 docstring = fix_docstring(docstring, indent)
 312             else:
 313                 docstring = docstring.strip()
 314
 315             if docstring:
 316                 # Add some padding if the docstring starts / ends with a quote mark.
 317                 if docstring[0] == quote_char:
 318                     docstring = " " + docstring
 319                 if docstring[-1] == quote_char:
 320                     docstring += " "
 321                 if docstring[-1] == "\\":
 322                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
 323                     if backslash_count % 2:
 324                         # Odd number of tailing backslashes, add some padding to
 325                         # avoid escaping the closing string quote.
 326                         docstring += " "
 327             elif not docstring_started_empty:
 328                 docstring = " "
 329
 330             # We could enforce triple quotes at this point.
 331             quote = quote_char * quote_len
 332
 333             # It's invalid to put closing single-character quotes on a new line.
 334             if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
 335                 # We need to find the length of the last line of the docstring
 336                 # to find if we can add the closing quotes to the line without
 337                 # exceeding the maximum line length.
 338                 # If docstring is one line, then we need to add the length
 339                 # of the indent, prefix, and starting quotes. Ending quotes are
 340                 # handled later.
 341                 lines = docstring.splitlines()
 342                 last_line_length = len(lines[-1]) if docstring else 0
 343
 344                 if len(lines) == 1:
 345                     last_line_length += len(indent) + len(prefix) + quote_len
 346
 347                 # If adding closing quotes would cause the last line to exceed
 348                 # the maximum line length then put a line break before the
 349                 # closing quotes
 350                 if last_line_length + quote_len > self.mode.line_length:
 351                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
 352                 else:
 353                     leaf.value = prefix + quote + docstring + quote
 354             else:
 355                 leaf.value = prefix + quote + docstring + quote
 356
 357         yield from self.visit_default(leaf)
 358
 359     def __post_init__(self) -> None:
 360         """You are in a twisty little maze of passages."""
 361         self.current_line = Line(mode=self.mode)
 362
 363         v = self.visit_stmt
 364         Ø: Set[str] = set()
 365         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
 366         self.visit_if_stmt = partial(
 367             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
 368         )
 369         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
 370         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
 371         self.visit_try_stmt = partial(
 372             v, keywords={"try", "except", "else", "finally"}, parens=Ø
 373         )
 374         if self.mode.preview:
 375             self.visit_except_clause = partial(
 376                 v, keywords={"except"}, parens={"except"}
 377             )
 378             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
 379         else:
 380             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
 381             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
 382         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
 383         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
 384         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
 385         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
 386         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
 387         self.visit_async_funcdef = self.visit_async_stmt
 388         self.visit_decorated = self.visit_decorators
 389
 390         # PEP 634
 391         self.visit_match_stmt = self.visit_match_case
 392         self.visit_case_block = self.visit_match_case
 393
 394
 395 def transform_line(
 396     line: Line, mode: Mode, features: Collection[Feature] = ()
 397 ) -> Iterator[Line]:
 398     """Transform a `line`, potentially splitting it into many lines.
 399
 400     They should fit in the allotted `line_length` but might not be able to.
 401
 402     `features` are syntactical features that may be used in the output.
 403     """
 404     if line.is_comment:
 405         yield line
 406         return
 407
 408     line_str = line_to_string(line)
 409
 410     ll = mode.line_length
 411     sn = mode.string_normalization
 412     string_merge = StringMerger(ll, sn)
 413     string_paren_strip = StringParenStripper(ll, sn)
 414     string_split = StringSplitter(ll, sn)
 415     string_paren_wrap = StringParenWrapper(ll, sn)
 416
 417     transformers: List[Transformer]
 418     if (
 419         not line.contains_uncollapsable_type_comments()
 420         and not line.should_split_rhs
 421         and not line.magic_trailing_comma
 422         and (
 423             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
 424             or line.contains_unsplittable_type_ignore()
 425         )
 426         and not (line.inside_brackets and line.contains_standalone_comments())
 427     ):
 428         # Only apply basic string preprocessing, since lines shouldn't be split here.
 429         if Preview.string_processing in mode:
 430             transformers = [string_merge, string_paren_strip]
 431         else:
 432             transformers = []
 433     elif line.is_def:
 434         transformers = [left_hand_split]
 435     else:
 436
 437         def _rhs(
 438             self: object, line: Line, features: Collection[Feature]
 439         ) -> Iterator[Line]:
 440             """Wraps calls to `right_hand_split`.
 441
 442             The calls increasingly `omit` right-hand trailers (bracket pairs with
 443             content), meaning the trailers get glued together to split on another
 444             bracket pair instead.
 445             """
 446             for omit in generate_trailers_to_omit(line, mode.line_length):
 447                 lines = list(
 448                     right_hand_split(line, mode.line_length, features, omit=omit)
 449                 )
 450                 # Note: this check is only able to figure out if the first line of the
 451                 # *current* transformation fits in the line length.  This is true only
 452                 # for simple cases.  All others require running more transforms via
 453                 # `transform_line()`.  This check doesn't know if those would succeed.
 454                 if is_line_short_enough(lines[0], line_length=mode.line_length):
 455                     yield from lines
 456                     return
 457
 458             # All splits failed, best effort split with no omits.
 459             # This mostly happens to multiline strings that are by definition
 460             # reported as not fitting a single line, as well as lines that contain
 461             # trailing commas (those have to be exploded).
 462             yield from right_hand_split(
 463                 line, line_length=mode.line_length, features=features
 464             )
 465
 466         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
 467         # __name__ attribute which is needed in `run_transformer` further down.
 468         # Unfortunately a nested class breaks mypyc too. So a class must be created
 469         # via type ... https://github.com/mypyc/mypyc/issues/884
 470         rhs = type("rhs", (), {"__call__": _rhs})()
 471
 472         if Preview.string_processing in mode:
 473             if line.inside_brackets:
 474                 transformers = [
 475                     string_merge,
 476                     string_paren_strip,
 477                     string_split,
 478                     delimiter_split,
 479                     standalone_comment_split,
 480                     string_paren_wrap,
 481                     rhs,
 482                 ]
 483             else:
 484                 transformers = [
 485                     string_merge,
 486                     string_paren_strip,
 487                     string_split,
 488                     string_paren_wrap,
 489                     rhs,
 490                 ]
 491         else:
 492             if line.inside_brackets:
 493                 transformers = [delimiter_split, standalone_comment_split, rhs]
 494             else:
 495                 transformers = [rhs]
 496     # It's always safe to attempt hugging of power operations and pretty much every line
 497     # could match.
 498     transformers.append(hug_power_op)
 499
 500     for transform in transformers:
 501         # We are accumulating lines in `result` because we might want to abort
 502         # mission and return the original line in the end, or attempt a different
 503         # split altogether.
 504         try:
 505             result = run_transformer(line, transform, mode, features, line_str=line_str)
 506         except CannotTransform:
 507             continue
 508         else:
 509             yield from result
 510             break
 511
 512     else:
 513         yield line
 514
 515
 516 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
 517     """Split line into many lines, starting with the first matching bracket pair.
 518
 519     Note: this usually looks weird, only use this for function definitions.
 520     Prefer RHS otherwise.  This is why this function is not symmetrical with
 521     :func:`right_hand_split` which also handles optional parentheses.
 522     """
 523     tail_leaves: List[Leaf] = []
 524     body_leaves: List[Leaf] = []
 525     head_leaves: List[Leaf] = []
 526     current_leaves = head_leaves
 527     matching_bracket: Optional[Leaf] = None
 528     for leaf in line.leaves:
 529         if (
 530             current_leaves is body_leaves
 531             and leaf.type in CLOSING_BRACKETS
 532             and leaf.opening_bracket is matching_bracket
 533             and isinstance(matching_bracket, Leaf)
 534         ):
 535             ensure_visible(leaf)
 536             ensure_visible(matching_bracket)
 537             current_leaves = tail_leaves if body_leaves else head_leaves
 538         current_leaves.append(leaf)
 539         if current_leaves is head_leaves:
 540             if leaf.type in OPENING_BRACKETS:
 541                 matching_bracket = leaf
 542                 current_leaves = body_leaves
 543     if not matching_bracket:
 544         raise CannotSplit("No brackets found")
 545
 546     head = bracket_split_build_line(head_leaves, line, matching_bracket)
 547     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
 548     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
 549     bracket_split_succeeded_or_raise(head, body, tail)
 550     for result in (head, body, tail):
 551         if result:
 552             yield result
 553
 554
 555 def right_hand_split(
 556     line: Line,
 557     line_length: int,
 558     features: Collection[Feature] = (),
 559     omit: Collection[LeafID] = (),
 560 ) -> Iterator[Line]:
 561     """Split line into many lines, starting with the last matching bracket pair.
 562
 563     If the split was by optional parentheses, attempt splitting without them, too.
 564     `omit` is a collection of closing bracket IDs that shouldn't be considered for
 565     this split.
 566
 567     Note: running this function modifies `bracket_depth` on the leaves of `line`.
 568     """
 569     tail_leaves: List[Leaf] = []
 570     body_leaves: List[Leaf] = []
 571     head_leaves: List[Leaf] = []
 572     current_leaves = tail_leaves
 573     opening_bracket: Optional[Leaf] = None
 574     closing_bracket: Optional[Leaf] = None
 575     for leaf in reversed(line.leaves):
 576         if current_leaves is body_leaves:
 577             if leaf is opening_bracket:
 578                 current_leaves = head_leaves if body_leaves else tail_leaves
 579         current_leaves.append(leaf)
 580         if current_leaves is tail_leaves:
 581             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
 582                 opening_bracket = leaf.opening_bracket
 583                 closing_bracket = leaf
 584                 current_leaves = body_leaves
 585     if not (opening_bracket and closing_bracket and head_leaves):
 586         # If there is no opening or closing_bracket that means the split failed and
 587         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
 588         # the matching `opening_bracket` wasn't available on `line` anymore.
 589         raise CannotSplit("No brackets found")
 590
 591     tail_leaves.reverse()
 592     body_leaves.reverse()
 593     head_leaves.reverse()
 594     head = bracket_split_build_line(head_leaves, line, opening_bracket)
 595     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
 596     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
 597     bracket_split_succeeded_or_raise(head, body, tail)
 598     if (
 599         Feature.FORCE_OPTIONAL_PARENTHESES not in features
 600         # the opening bracket is an optional paren
 601         and opening_bracket.type == token.LPAR
 602         and not opening_bracket.value
 603         # the closing bracket is an optional paren
 604         and closing_bracket.type == token.RPAR
 605         and not closing_bracket.value
 606         # it's not an import (optional parens are the only thing we can split on
 607         # in this case; attempting a split without them is a waste of time)
 608         and not line.is_import
 609         # there are no standalone comments in the body
 610         and not body.contains_standalone_comments(0)
 611         # and we can actually remove the parens
 612         and can_omit_invisible_parens(body, line_length)
 613     ):
 614         omit = {id(closing_bracket), *omit}
 615         try:
 616             yield from right_hand_split(line, line_length, features=features, omit=omit)
 617             return
 618
 619         except CannotSplit as e:
 620             if not (
 621                 can_be_split(body)
 622                 or is_line_short_enough(body, line_length=line_length)
 623             ):
 624                 raise CannotSplit(
 625                     "Splitting failed, body is still too long and can't be split."
 626                 ) from e
 627
 628             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
 629                 raise CannotSplit(
 630                     "The current optional pair of parentheses is bound to fail to"
 631                     " satisfy the splitting algorithm because the head or the tail"
 632                     " contains multiline strings which by definition never fit one"
 633                     " line."
 634                 ) from e
 635
 636     ensure_visible(opening_bracket)
 637     ensure_visible(closing_bracket)
 638     for result in (head, body, tail):
 639         if result:
 640             yield result
 641
 642
 643 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
 644     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
 645
 646     Do nothing otherwise.
 647
 648     A left- or right-hand split is based on a pair of brackets. Content before
 649     (and including) the opening bracket is left on one line, content inside the
 650     brackets is put on a separate line, and finally content starting with and
 651     following the closing bracket is put on a separate line.
 652
 653     Those are called `head`, `body`, and `tail`, respectively. If the split
 654     produced the same line (all content in `head`) or ended up with an empty `body`
 655     and the `tail` is just the closing bracket, then it's considered failed.
 656     """
 657     tail_len = len(str(tail).strip())
 658     if not body:
 659         if tail_len == 0:
 660             raise CannotSplit("Splitting brackets produced the same line")
 661
 662         elif tail_len < 3:
 663             raise CannotSplit(
 664                 f"Splitting brackets on an empty body to save {tail_len} characters is"
 665                 " not worth it"
 666             )
 667
 668
 669 def bracket_split_build_line(
 670     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
 671 ) -> Line:
 672     """Return a new line with given `leaves` and respective comments from `original`.
 673
 674     If `is_body` is True, the result line is one-indented inside brackets and as such
 675     has its first leaf's prefix normalized and a trailing comma added when expected.
 676     """
 677     result = Line(mode=original.mode, depth=original.depth)
 678     if is_body:
 679         result.inside_brackets = True
 680         result.depth += 1
 681         if leaves:
 682             # Since body is a new indent level, remove spurious leading whitespace.
 683             normalize_prefix(leaves[0], inside_brackets=True)
 684             # Ensure a trailing comma for imports and standalone function arguments, but
 685             # be careful not to add one after any comments or within type annotations.
 686             no_commas = (
 687                 original.is_def
 688                 and opening_bracket.value == "("
 689                 and not any(leaf.type == token.COMMA for leaf in leaves)
 690                 # In particular, don't add one within a parenthesized return annotation.
 691                 # Unfortunately the indicator we're in a return annotation (RARROW) may
 692                 # be defined directly in the parent node, the parent of the parent ...
 693                 # and so on depending on how complex the return annotation is.
 694                 # This isn't perfect and there's some false negatives but they are in
 695                 # contexts were a comma is actually fine.
 696                 and not any(
 697                     node.prev_sibling.type == RARROW
 698                     for node in (
 699                         leaves[0].parent,
 700                         getattr(leaves[0].parent, "parent", None),
 701                     )
 702                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
 703                 )
 704             )
 705
 706             if original.is_import or no_commas:
 707                 for i in range(len(leaves) - 1, -1, -1):
 708                     if leaves[i].type == STANDALONE_COMMENT:
 709                         continue
 710
 711                     if leaves[i].type != token.COMMA:
 712                         new_comma = Leaf(token.COMMA, ",")
 713                         leaves.insert(i + 1, new_comma)
 714                     break
 715
 716     # Populate the line
 717     for leaf in leaves:
 718         result.append(leaf, preformatted=True)
 719         for comment_after in original.comments_after(leaf):
 720             result.append(comment_after, preformatted=True)
 721     if is_body and should_split_line(result, opening_bracket):
 722         result.should_split_rhs = True
 723     return result
 724
 725
 726 def dont_increase_indentation(split_func: Transformer) -> Transformer:
 727     """Normalize prefix of the first leaf in every line returned by `split_func`.
 728
 729     This is a decorator over relevant split functions.
 730     """
 731
 732     @wraps(split_func)
 733     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 734         for split_line in split_func(line, features):
 735             normalize_prefix(split_line.leaves[0], inside_brackets=True)
 736             yield split_line
 737
 738     return split_wrapper
 739
 740
 741 @dont_increase_indentation
 742 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
 743     """Split according to delimiters of the highest priority.
 744
 745     If the appropriate Features are given, the split will add trailing commas
 746     also in function signatures and calls that contain `*` and `**`.
 747     """
 748     try:
 749         last_leaf = line.leaves[-1]
 750     except IndexError:
 751         raise CannotSplit("Line empty") from None
 752
 753     bt = line.bracket_tracker
 754     try:
 755         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
 756     except ValueError:
 757         raise CannotSplit("No delimiters found") from None
 758
 759     if delimiter_priority == DOT_PRIORITY:
 760         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
 761             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
 762
 763     current_line = Line(
 764         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 765     )
 766     lowest_depth = sys.maxsize
 767     trailing_comma_safe = True
 768
 769     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 770         """Append `leaf` to current line or to new line if appending impossible."""
 771         nonlocal current_line
 772         try:
 773             current_line.append_safe(leaf, preformatted=True)
 774         except ValueError:
 775             yield current_line
 776
 777             current_line = Line(
 778                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 779             )
 780             current_line.append(leaf)
 781
 782     for leaf in line.leaves:
 783         yield from append_to_line(leaf)
 784
 785         for comment_after in line.comments_after(leaf):
 786             yield from append_to_line(comment_after)
 787
 788         lowest_depth = min(lowest_depth, leaf.bracket_depth)
 789         if leaf.bracket_depth == lowest_depth:
 790             if is_vararg(leaf, within={syms.typedargslist}):
 791                 trailing_comma_safe = (
 792                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
 793                 )
 794             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
 795                 trailing_comma_safe = (
 796                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
 797                 )
 798
 799         leaf_priority = bt.delimiters.get(id(leaf))
 800         if leaf_priority == delimiter_priority:
 801             yield current_line
 802
 803             current_line = Line(
 804                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 805             )
 806     if current_line:
 807         if (
 808             trailing_comma_safe
 809             and delimiter_priority == COMMA_PRIORITY
 810             and current_line.leaves[-1].type != token.COMMA
 811             and current_line.leaves[-1].type != STANDALONE_COMMENT
 812         ):
 813             new_comma = Leaf(token.COMMA, ",")
 814             current_line.append(new_comma)
 815         yield current_line
 816
 817
 818 @dont_increase_indentation
 819 def standalone_comment_split(
 820     line: Line, features: Collection[Feature] = ()
 821 ) -> Iterator[Line]:
 822     """Split standalone comments from the rest of the line."""
 823     if not line.contains_standalone_comments(0):
 824         raise CannotSplit("Line does not have any standalone comments")
 825
 826     current_line = Line(
 827         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 828     )
 829
 830     def append_to_line(leaf: Leaf) -> Iterator[Line]:
 831         """Append `leaf` to current line or to new line if appending impossible."""
 832         nonlocal current_line
 833         try:
 834             current_line.append_safe(leaf, preformatted=True)
 835         except ValueError:
 836             yield current_line
 837
 838             current_line = Line(
 839                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
 840             )
 841             current_line.append(leaf)
 842
 843     for leaf in line.leaves:
 844         yield from append_to_line(leaf)
 845
 846         for comment_after in line.comments_after(leaf):
 847             yield from append_to_line(comment_after)
 848
 849     if current_line:
 850         yield current_line
 851
 852
 853 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
 854     """Leave existing extra newlines if not `inside_brackets`. Remove everything
 855     else.
 856
 857     Note: don't use backslashes for formatting or you'll lose your voting rights.
 858     """
 859     if not inside_brackets:
 860         spl = leaf.prefix.split("#")
 861         if "\\" not in spl[0]:
 862             nl_count = spl[-1].count("\n")
 863             if len(spl) > 1:
 864                 nl_count -= 1
 865             leaf.prefix = "\n" * nl_count
 866             return
 867
 868     leaf.prefix = ""
 869
 870
 871 def normalize_invisible_parens(
 872     node: Node, parens_after: Set[str], *, preview: bool
 873 ) -> None:
 874     """Make existing optional parentheses invisible or create new ones.
 875
 876     `parens_after` is a set of string leaf values immediately after which parens
 877     should be put.
 878
 879     Standardizes on visible parentheses for single-element tuples, and keeps
 880     existing visible parentheses for other tuples and generator expressions.
 881     """
 882     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
 883         if pc.value in FMT_OFF:
 884             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
 885             return
 886     check_lpar = False
 887     for index, child in enumerate(list(node.children)):
 888         # Fixes a bug where invisible parens are not properly stripped from
 889         # assignment statements that contain type annotations.
 890         if isinstance(child, Node) and child.type == syms.annassign:
 891             normalize_invisible_parens(
 892                 child, parens_after=parens_after, preview=preview
 893             )
 894
 895         # Add parentheses around long tuple unpacking in assignments.
 896         if (
 897             index == 0
 898             and isinstance(child, Node)
 899             and child.type == syms.testlist_star_expr
 900         ):
 901             check_lpar = True
 902
 903         if check_lpar:
 904             if (
 905                 preview
 906                 and child.type == syms.atom
 907                 and node.type == syms.for_stmt
 908                 and isinstance(child.prev_sibling, Leaf)
 909                 and child.prev_sibling.type == token.NAME
 910                 and child.prev_sibling.value == "for"
 911             ):
 912                 if maybe_make_parens_invisible_in_atom(
 913                     child,
 914                     parent=node,
 915                     remove_brackets_around_comma=True,
 916                 ):
 917                     wrap_in_parentheses(node, child, visible=False)
 918             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
 919                 remove_with_parens(child, node)
 920             elif child.type == syms.atom:
 921                 if maybe_make_parens_invisible_in_atom(
 922                     child,
 923                     parent=node,
 924                 ):
 925                     wrap_in_parentheses(node, child, visible=False)
 926             elif is_one_tuple(child):
 927                 wrap_in_parentheses(node, child, visible=True)
 928             elif node.type == syms.import_from:
 929                 # "import from" nodes store parentheses directly as part of
 930                 # the statement
 931                 if is_lpar_token(child):
 932                     assert is_rpar_token(node.children[-1])
 933                     # make parentheses invisible
 934                     child.value = ""
 935                     node.children[-1].value = ""
 936                 elif child.type != token.STAR:
 937                     # insert invisible parentheses
 938                     node.insert_child(index, Leaf(token.LPAR, ""))
 939                     node.append_child(Leaf(token.RPAR, ""))
 940                 break
 941             elif (
 942                 index == 1
 943                 and child.type == token.STAR
 944                 and node.type == syms.except_clause
 945             ):
 946                 # In except* (PEP 654), the star is actually part of
 947                 # of the keyword. So we need to skip the insertion of
 948                 # invisible parentheses to work more precisely.
 949                 continue
 950
 951             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
 952                 wrap_in_parentheses(node, child, visible=False)
 953
 954         comma_check = child.type == token.COMMA if preview else False
 955
 956         check_lpar = isinstance(child, Leaf) and (
 957             child.value in parens_after or comma_check
 958         )
 959
 960
 961 def remove_await_parens(node: Node) -> None:
 962     if node.children[0].type == token.AWAIT and len(node.children) > 1:
 963         if (
 964             node.children[1].type == syms.atom
 965             and node.children[1].children[0].type == token.LPAR
 966         ):
 967             if maybe_make_parens_invisible_in_atom(
 968                 node.children[1],
 969                 parent=node,
 970                 remove_brackets_around_comma=True,
 971             ):
 972                 wrap_in_parentheses(node, node.children[1], visible=False)
 973
 974             # Since await is an expression we shouldn't remove
 975             # brackets in cases where this would change
 976             # the AST due to operator precedence.
 977             # Therefore we only aim to remove brackets around
 978             # power nodes that aren't also await expressions themselves.
 979             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
 980             # N.B. We've still removed any redundant nested brackets though :)
 981             opening_bracket = cast(Leaf, node.children[1].children[0])
 982             closing_bracket = cast(Leaf, node.children[1].children[-1])
 983             bracket_contents = cast(Node, node.children[1].children[1])
 984             if bracket_contents.type != syms.power:
 985                 ensure_visible(opening_bracket)
 986                 ensure_visible(closing_bracket)
 987             elif (
 988                 bracket_contents.type == syms.power
 989                 and bracket_contents.children[0].type == token.AWAIT
 990             ):
 991                 ensure_visible(opening_bracket)
 992                 ensure_visible(closing_bracket)
 993                 # If we are in a nested await then recurse down.
 994                 remove_await_parens(bracket_contents)
 995
 996
 997 def remove_with_parens(node: Node, parent: Node) -> None:
 998     """Recursively hide optional parens in `with` statements."""
 999     # Removing all unnecessary parentheses in with statements in one pass is a tad
1000     # complex as different variations of bracketed statements result in pretty
1001     # different parse trees:
1002     #
1003     # with (open("file")) as f:                       # this is an asexpr_test
1004     #     ...
1005     #
1006     # with (open("file") as f):                       # this is an atom containing an
1007     #     ...                                         # asexpr_test
1008     #
1009     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1010     #     ...                                         # asexpr_test
1011     #
1012     # with (open("file") as f, open("file") as f):    # an atom containing a
1013     #     ...                                         # testlist_gexp which then
1014     #                                                 # contains multiple asexpr_test(s)
1015     if node.type == syms.atom:
1016         if maybe_make_parens_invisible_in_atom(
1017             node,
1018             parent=parent,
1019             remove_brackets_around_comma=True,
1020         ):
1021             wrap_in_parentheses(parent, node, visible=False)
1022         if isinstance(node.children[1], Node):
1023             remove_with_parens(node.children[1], node)
1024     elif node.type == syms.testlist_gexp:
1025         for child in node.children:
1026             if isinstance(child, Node):
1027                 remove_with_parens(child, node)
1028     elif node.type == syms.asexpr_test and not any(
1029         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1030     ):
1031         if maybe_make_parens_invisible_in_atom(
1032             node.children[0],
1033             parent=node,
1034             remove_brackets_around_comma=True,
1035         ):
1036             wrap_in_parentheses(node, node.children[0], visible=False)
1037
1038
1039 def maybe_make_parens_invisible_in_atom(
1040     node: LN,
1041     parent: LN,
1042     remove_brackets_around_comma: bool = False,
1043 ) -> bool:
1044     """If it's safe, make the parens in the atom `node` invisible, recursively.
1045     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1046     as they are redundant.
1047
1048     Returns whether the node should itself be wrapped in invisible parentheses.
1049     """
1050     if (
1051         node.type != syms.atom
1052         or is_empty_tuple(node)
1053         or is_one_tuple(node)
1054         or (is_yield(node) and parent.type != syms.expr_stmt)
1055         or (
1056             # This condition tries to prevent removing non-optional brackets
1057             # around a tuple, however, can be a bit overzealous so we provide
1058             # and option to skip this check for `for` and `with` statements.
1059             not remove_brackets_around_comma
1060             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1061         )
1062     ):
1063         return False
1064
1065     if is_walrus_assignment(node):
1066         if parent.type in [
1067             syms.annassign,
1068             syms.expr_stmt,
1069             syms.assert_stmt,
1070             syms.return_stmt,
1071             # these ones aren't useful to end users, but they do please fuzzers
1072             syms.for_stmt,
1073             syms.del_stmt,
1074         ]:
1075             return False
1076
1077     first = node.children[0]
1078     last = node.children[-1]
1079     if is_lpar_token(first) and is_rpar_token(last):
1080         middle = node.children[1]
1081         # make parentheses invisible
1082         first.value = ""
1083         last.value = ""
1084         maybe_make_parens_invisible_in_atom(
1085             middle,
1086             parent=parent,
1087             remove_brackets_around_comma=remove_brackets_around_comma,
1088         )
1089
1090         if is_atom_with_invisible_parens(middle):
1091             # Strip the invisible parens from `middle` by replacing
1092             # it with the child in-between the invisible parens
1093             middle.replace(middle.children[1])
1094
1095         return False
1096
1097     return True
1098
1099
1100 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1101     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1102
1103     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1104         return False
1105
1106     # We're essentially checking if the body is delimited by commas and there's more
1107     # than one of them (we're excluding the trailing comma and if the delimiter priority
1108     # is still commas, that means there's more).
1109     exclude = set()
1110     trailing_comma = False
1111     try:
1112         last_leaf = line.leaves[-1]
1113         if last_leaf.type == token.COMMA:
1114             trailing_comma = True
1115             exclude.add(id(last_leaf))
1116         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1117     except (IndexError, ValueError):
1118         return False
1119
1120     return max_priority == COMMA_PRIORITY and (
1121         (line.mode.magic_trailing_comma and trailing_comma)
1122         # always explode imports
1123         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1124     )
1125
1126
1127 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1128     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1129
1130     Brackets can be omitted if the entire trailer up to and including
1131     a preceding closing bracket fits in one line.
1132
1133     Yielded sets are cumulative (contain results of previous yields, too).  First
1134     set is empty, unless the line should explode, in which case bracket pairs until
1135     the one that needs to explode are omitted.
1136     """
1137
1138     omit: Set[LeafID] = set()
1139     if not line.magic_trailing_comma:
1140         yield omit
1141
1142     length = 4 * line.depth
1143     opening_bracket: Optional[Leaf] = None
1144     closing_bracket: Optional[Leaf] = None
1145     inner_brackets: Set[LeafID] = set()
1146     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1147         length += leaf_length
1148         if length > line_length:
1149             break
1150
1151         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1152         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1153             break
1154
1155         if opening_bracket:
1156             if leaf is opening_bracket:
1157                 opening_bracket = None
1158             elif leaf.type in CLOSING_BRACKETS:
1159                 prev = line.leaves[index - 1] if index > 0 else None
1160                 if (
1161                     prev
1162                     and prev.type == token.COMMA
1163                     and leaf.opening_bracket is not None
1164                     and not is_one_sequence_between(
1165                         leaf.opening_bracket, leaf, line.leaves
1166                     )
1167                 ):
1168                     # Never omit bracket pairs with trailing commas.
1169                     # We need to explode on those.
1170                     break
1171
1172                 inner_brackets.add(id(leaf))
1173         elif leaf.type in CLOSING_BRACKETS:
1174             prev = line.leaves[index - 1] if index > 0 else None
1175             if prev and prev.type in OPENING_BRACKETS:
1176                 # Empty brackets would fail a split so treat them as "inner"
1177                 # brackets (e.g. only add them to the `omit` set if another
1178                 # pair of brackets was good enough.
1179                 inner_brackets.add(id(leaf))
1180                 continue
1181
1182             if closing_bracket:
1183                 omit.add(id(closing_bracket))
1184                 omit.update(inner_brackets)
1185                 inner_brackets.clear()
1186                 yield omit
1187
1188             if (
1189                 prev
1190                 and prev.type == token.COMMA
1191                 and leaf.opening_bracket is not None
1192                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1193             ):
1194                 # Never omit bracket pairs with trailing commas.
1195                 # We need to explode on those.
1196                 break
1197
1198             if leaf.value:
1199                 opening_bracket = leaf.opening_bracket
1200                 closing_bracket = leaf
1201
1202
1203 def run_transformer(
1204     line: Line,
1205     transform: Transformer,
1206     mode: Mode,
1207     features: Collection[Feature],
1208     *,
1209     line_str: str = "",
1210 ) -> List[Line]:
1211     if not line_str:
1212         line_str = line_to_string(line)
1213     result: List[Line] = []
1214     for transformed_line in transform(line, features):
1215         if str(transformed_line).strip("\n") == line_str:
1216             raise CannotTransform("Line transformer returned an unchanged result")
1217
1218         result.extend(transform_line(transformed_line, mode=mode, features=features))
1219
1220     if (
1221         transform.__class__.__name__ != "rhs"
1222         or not line.bracket_tracker.invisible
1223         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1224         or line.contains_multiline_strings()
1225         or result[0].contains_uncollapsable_type_comments()
1226         or result[0].contains_unsplittable_type_ignore()
1227         or is_line_short_enough(result[0], line_length=mode.line_length)
1228         # If any leaves have no parents (which _can_ occur since
1229         # `transform(line)` potentially destroys the line's underlying node
1230         # structure), then we can't proceed. Doing so would cause the below
1231         # call to `append_leaves()` to fail.
1232         or any(leaf.parent is None for leaf in line.leaves)
1233     ):
1234         return result
1235
1236     line_copy = line.clone()
1237     append_leaves(line_copy, line, line.leaves)
1238     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1239     second_opinion = run_transformer(
1240         line_copy, transform, mode, features_fop, line_str=line_str
1241     )
1242     if all(
1243         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1244     ):
1245         result = second_opinion
1246     return result