]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

fix: allow tests to be run from (hopefully) any directory (GH-2574)
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 from functools import partial, wraps
5 import sys
6 from typing import Collection, Iterator, List, Optional, Set, Union
7
8 from dataclasses import dataclass, field
9
10 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
11 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
12 from black.nodes import Visitor, syms, first_child_is_arith, ensure_visible
13 from black.nodes import is_docstring, is_empty_tuple, is_one_tuple, is_one_tuple_between
14 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
15 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
16 from black.nodes import wrap_in_parentheses
17 from black.brackets import max_delimiter_priority_in_atom
18 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
19 from black.lines import Line, line_to_string, is_line_short_enough
20 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
21 from black.comments import generate_comments, list_comments, FMT_OFF
22 from black.numerics import normalize_numeric_literal
23 from black.strings import get_string_prefix, fix_docstring
24 from black.strings import normalize_string_prefix, normalize_string_quotes
25 from black.trans import Transformer, CannotTransform, StringMerger
26 from black.trans import StringSplitter, StringParenWrapper, StringParenStripper
27 from black.mode import Mode
28 from black.mode import Feature
29
30 from blib2to3.pytree import Node, Leaf
31 from blib2to3.pgen2 import token
32
33
34 # types
35 LeafID = int
36 LN = Union[Leaf, Node]
37
38
39 class CannotSplit(CannotTransform):
40     """A readable split that fits the allotted line length is impossible."""
41
42
43 @dataclass
44 class LineGenerator(Visitor[Line]):
45     """Generates reformatted Line objects.  Empty lines are not emitted.
46
47     Note: destroys the tree it's visiting by mutating prefixes of its leaves
48     in ways that will no longer stringify to valid Python code on the tree.
49     """
50
51     mode: Mode
52     remove_u_prefix: bool = False
53     current_line: Line = field(init=False)
54
55     def line(self, indent: int = 0) -> Iterator[Line]:
56         """Generate a line.
57
58         If the line is empty, only emit if it makes sense.
59         If the line is too long, split it first and then generate.
60
61         If any lines were generated, set up a new current_line.
62         """
63         if not self.current_line:
64             self.current_line.depth += indent
65             return  # Line is empty, don't emit. Creating a new one unnecessary.
66
67         complete_line = self.current_line
68         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
69         yield complete_line
70
71     def visit_default(self, node: LN) -> Iterator[Line]:
72         """Default `visit_*()` implementation. Recurses to children of `node`."""
73         if isinstance(node, Leaf):
74             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
75             for comment in generate_comments(node):
76                 if any_open_brackets:
77                     # any comment within brackets is subject to splitting
78                     self.current_line.append(comment)
79                 elif comment.type == token.COMMENT:
80                     # regular trailing comment
81                     self.current_line.append(comment)
82                     yield from self.line()
83
84                 else:
85                     # regular standalone comment
86                     yield from self.line()
87
88                     self.current_line.append(comment)
89                     yield from self.line()
90
91             normalize_prefix(node, inside_brackets=any_open_brackets)
92             if self.mode.string_normalization and node.type == token.STRING:
93                 node.value = normalize_string_prefix(
94                     node.value, remove_u_prefix=self.remove_u_prefix
95                 )
96                 node.value = normalize_string_quotes(node.value)
97             if node.type == token.NUMBER:
98                 normalize_numeric_literal(node)
99             if node.type not in WHITESPACE:
100                 self.current_line.append(node)
101         yield from super().visit_default(node)
102
103     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
104         """Increase indentation level, maybe yield a line."""
105         # In blib2to3 INDENT never holds comments.
106         yield from self.line(+1)
107         yield from self.visit_default(node)
108
109     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
110         """Decrease indentation level, maybe yield a line."""
111         # The current line might still wait for trailing comments.  At DEDENT time
112         # there won't be any (they would be prefixes on the preceding NEWLINE).
113         # Emit the line then.
114         yield from self.line()
115
116         # While DEDENT has no value, its prefix may contain standalone comments
117         # that belong to the current indentation level.  Get 'em.
118         yield from self.visit_default(node)
119
120         # Finally, emit the dedent.
121         yield from self.line(-1)
122
123     def visit_stmt(
124         self, node: Node, keywords: Set[str], parens: Set[str]
125     ) -> Iterator[Line]:
126         """Visit a statement.
127
128         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
129         `def`, `with`, `class`, `assert` and assignments.
130
131         The relevant Python language `keywords` for a given statement will be
132         NAME leaves within it. This methods puts those on a separate line.
133
134         `parens` holds a set of string leaf values immediately after which
135         invisible parens should be put.
136         """
137         normalize_invisible_parens(node, parens_after=parens)
138         for child in node.children:
139             if child.type == token.NAME and child.value in keywords:  # type: ignore
140                 yield from self.line()
141
142             yield from self.visit(child)
143
144     def visit_suite(self, node: Node) -> Iterator[Line]:
145         """Visit a suite."""
146         if self.mode.is_pyi and is_stub_suite(node):
147             yield from self.visit(node.children[2])
148         else:
149             yield from self.visit_default(node)
150
151     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
152         """Visit a statement without nested statements."""
153         if first_child_is_arith(node):
154             wrap_in_parentheses(node, node.children[0], visible=False)
155         is_suite_like = node.parent and node.parent.type in STATEMENT
156         if is_suite_like:
157             if self.mode.is_pyi and is_stub_body(node):
158                 yield from self.visit_default(node)
159             else:
160                 yield from self.line(+1)
161                 yield from self.visit_default(node)
162                 yield from self.line(-1)
163
164         else:
165             if (
166                 not self.mode.is_pyi
167                 or not node.parent
168                 or not is_stub_suite(node.parent)
169             ):
170                 yield from self.line()
171             yield from self.visit_default(node)
172
173     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
174         """Visit `async def`, `async for`, `async with`."""
175         yield from self.line()
176
177         children = iter(node.children)
178         for child in children:
179             yield from self.visit(child)
180
181             if child.type == token.ASYNC:
182                 break
183
184         internal_stmt = next(children)
185         for child in internal_stmt.children:
186             yield from self.visit(child)
187
188     def visit_decorators(self, node: Node) -> Iterator[Line]:
189         """Visit decorators."""
190         for child in node.children:
191             yield from self.line()
192             yield from self.visit(child)
193
194     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
195         """Remove a semicolon and put the other statement on a separate line."""
196         yield from self.line()
197
198     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
199         """End of file. Process outstanding comments and end with a newline."""
200         yield from self.visit_default(leaf)
201         yield from self.line()
202
203     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
204         if not self.current_line.bracket_tracker.any_open_brackets():
205             yield from self.line()
206         yield from self.visit_default(leaf)
207
208     def visit_factor(self, node: Node) -> Iterator[Line]:
209         """Force parentheses between a unary op and a binary power:
210
211         -2 ** 8 -> -(2 ** 8)
212         """
213         _operator, operand = node.children
214         if (
215             operand.type == syms.power
216             and len(operand.children) == 3
217             and operand.children[1].type == token.DOUBLESTAR
218         ):
219             lpar = Leaf(token.LPAR, "(")
220             rpar = Leaf(token.RPAR, ")")
221             index = operand.remove() or 0
222             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
223         yield from self.visit_default(node)
224
225     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
226         if is_docstring(leaf) and "\\\n" not in leaf.value:
227             # We're ignoring docstrings with backslash newline escapes because changing
228             # indentation of those changes the AST representation of the code.
229             docstring = normalize_string_prefix(leaf.value, self.remove_u_prefix)
230             prefix = get_string_prefix(docstring)
231             docstring = docstring[len(prefix) :]  # Remove the prefix
232             quote_char = docstring[0]
233             # A natural way to remove the outer quotes is to do:
234             #   docstring = docstring.strip(quote_char)
235             # but that breaks on """""x""" (which is '""x').
236             # So we actually need to remove the first character and the next two
237             # characters but only if they are the same as the first.
238             quote_len = 1 if docstring[1] != quote_char else 3
239             docstring = docstring[quote_len:-quote_len]
240             docstring_started_empty = not docstring
241
242             if is_multiline_string(leaf):
243                 indent = " " * 4 * self.current_line.depth
244                 docstring = fix_docstring(docstring, indent)
245             else:
246                 docstring = docstring.strip()
247
248             if docstring:
249                 # Add some padding if the docstring starts / ends with a quote mark.
250                 if docstring[0] == quote_char:
251                     docstring = " " + docstring
252                 if docstring[-1] == quote_char:
253                     docstring += " "
254                 if docstring[-1] == "\\":
255                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
256                     if backslash_count % 2:
257                         # Odd number of tailing backslashes, add some padding to
258                         # avoid escaping the closing string quote.
259                         docstring += " "
260             elif not docstring_started_empty:
261                 docstring = " "
262
263             # We could enforce triple quotes at this point.
264             quote = quote_char * quote_len
265             leaf.value = prefix + quote + docstring + quote
266
267         yield from self.visit_default(leaf)
268
269     def __post_init__(self) -> None:
270         """You are in a twisty little maze of passages."""
271         self.current_line = Line(mode=self.mode)
272
273         v = self.visit_stmt
274         Ø: Set[str] = set()
275         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
276         self.visit_if_stmt = partial(
277             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
278         )
279         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
280         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
281         self.visit_try_stmt = partial(
282             v, keywords={"try", "except", "else", "finally"}, parens=Ø
283         )
284         self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
285         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
286         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
287         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
288         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
289         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
290         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
291         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
292         self.visit_async_funcdef = self.visit_async_stmt
293         self.visit_decorated = self.visit_decorators
294
295
296 def transform_line(
297     line: Line, mode: Mode, features: Collection[Feature] = ()
298 ) -> Iterator[Line]:
299     """Transform a `line`, potentially splitting it into many lines.
300
301     They should fit in the allotted `line_length` but might not be able to.
302
303     `features` are syntactical features that may be used in the output.
304     """
305     if line.is_comment:
306         yield line
307         return
308
309     line_str = line_to_string(line)
310
311     ll = mode.line_length
312     sn = mode.string_normalization
313     string_merge = StringMerger(ll, sn)
314     string_paren_strip = StringParenStripper(ll, sn)
315     string_split = StringSplitter(ll, sn)
316     string_paren_wrap = StringParenWrapper(ll, sn)
317
318     transformers: List[Transformer]
319     if (
320         not line.contains_uncollapsable_type_comments()
321         and not line.should_split_rhs
322         and not line.magic_trailing_comma
323         and (
324             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
325             or line.contains_unsplittable_type_ignore()
326         )
327         and not (line.inside_brackets and line.contains_standalone_comments())
328     ):
329         # Only apply basic string preprocessing, since lines shouldn't be split here.
330         if mode.experimental_string_processing:
331             transformers = [string_merge, string_paren_strip]
332         else:
333             transformers = []
334     elif line.is_def:
335         transformers = [left_hand_split]
336     else:
337
338         def rhs(line: Line, features: Collection[Feature]) -> Iterator[Line]:
339             """Wraps calls to `right_hand_split`.
340
341             The calls increasingly `omit` right-hand trailers (bracket pairs with
342             content), meaning the trailers get glued together to split on another
343             bracket pair instead.
344             """
345             for omit in generate_trailers_to_omit(line, mode.line_length):
346                 lines = list(
347                     right_hand_split(line, mode.line_length, features, omit=omit)
348                 )
349                 # Note: this check is only able to figure out if the first line of the
350                 # *current* transformation fits in the line length.  This is true only
351                 # for simple cases.  All others require running more transforms via
352                 # `transform_line()`.  This check doesn't know if those would succeed.
353                 if is_line_short_enough(lines[0], line_length=mode.line_length):
354                     yield from lines
355                     return
356
357             # All splits failed, best effort split with no omits.
358             # This mostly happens to multiline strings that are by definition
359             # reported as not fitting a single line, as well as lines that contain
360             # trailing commas (those have to be exploded).
361             yield from right_hand_split(
362                 line, line_length=mode.line_length, features=features
363             )
364
365         if mode.experimental_string_processing:
366             if line.inside_brackets:
367                 transformers = [
368                     string_merge,
369                     string_paren_strip,
370                     string_split,
371                     delimiter_split,
372                     standalone_comment_split,
373                     string_paren_wrap,
374                     rhs,
375                 ]
376             else:
377                 transformers = [
378                     string_merge,
379                     string_paren_strip,
380                     string_split,
381                     string_paren_wrap,
382                     rhs,
383                 ]
384         else:
385             if line.inside_brackets:
386                 transformers = [delimiter_split, standalone_comment_split, rhs]
387             else:
388                 transformers = [rhs]
389
390     for transform in transformers:
391         # We are accumulating lines in `result` because we might want to abort
392         # mission and return the original line in the end, or attempt a different
393         # split altogether.
394         try:
395             result = run_transformer(line, transform, mode, features, line_str=line_str)
396         except CannotTransform:
397             continue
398         else:
399             yield from result
400             break
401
402     else:
403         yield line
404
405
406 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
407     """Split line into many lines, starting with the first matching bracket pair.
408
409     Note: this usually looks weird, only use this for function definitions.
410     Prefer RHS otherwise.  This is why this function is not symmetrical with
411     :func:`right_hand_split` which also handles optional parentheses.
412     """
413     tail_leaves: List[Leaf] = []
414     body_leaves: List[Leaf] = []
415     head_leaves: List[Leaf] = []
416     current_leaves = head_leaves
417     matching_bracket: Optional[Leaf] = None
418     for leaf in line.leaves:
419         if (
420             current_leaves is body_leaves
421             and leaf.type in CLOSING_BRACKETS
422             and leaf.opening_bracket is matching_bracket
423         ):
424             current_leaves = tail_leaves if body_leaves else head_leaves
425         current_leaves.append(leaf)
426         if current_leaves is head_leaves:
427             if leaf.type in OPENING_BRACKETS:
428                 matching_bracket = leaf
429                 current_leaves = body_leaves
430     if not matching_bracket:
431         raise CannotSplit("No brackets found")
432
433     head = bracket_split_build_line(head_leaves, line, matching_bracket)
434     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
435     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
436     bracket_split_succeeded_or_raise(head, body, tail)
437     for result in (head, body, tail):
438         if result:
439             yield result
440
441
442 def right_hand_split(
443     line: Line,
444     line_length: int,
445     features: Collection[Feature] = (),
446     omit: Collection[LeafID] = (),
447 ) -> Iterator[Line]:
448     """Split line into many lines, starting with the last matching bracket pair.
449
450     If the split was by optional parentheses, attempt splitting without them, too.
451     `omit` is a collection of closing bracket IDs that shouldn't be considered for
452     this split.
453
454     Note: running this function modifies `bracket_depth` on the leaves of `line`.
455     """
456     tail_leaves: List[Leaf] = []
457     body_leaves: List[Leaf] = []
458     head_leaves: List[Leaf] = []
459     current_leaves = tail_leaves
460     opening_bracket: Optional[Leaf] = None
461     closing_bracket: Optional[Leaf] = None
462     for leaf in reversed(line.leaves):
463         if current_leaves is body_leaves:
464             if leaf is opening_bracket:
465                 current_leaves = head_leaves if body_leaves else tail_leaves
466         current_leaves.append(leaf)
467         if current_leaves is tail_leaves:
468             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
469                 opening_bracket = leaf.opening_bracket
470                 closing_bracket = leaf
471                 current_leaves = body_leaves
472     if not (opening_bracket and closing_bracket and head_leaves):
473         # If there is no opening or closing_bracket that means the split failed and
474         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
475         # the matching `opening_bracket` wasn't available on `line` anymore.
476         raise CannotSplit("No brackets found")
477
478     tail_leaves.reverse()
479     body_leaves.reverse()
480     head_leaves.reverse()
481     head = bracket_split_build_line(head_leaves, line, opening_bracket)
482     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
483     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
484     bracket_split_succeeded_or_raise(head, body, tail)
485     if (
486         Feature.FORCE_OPTIONAL_PARENTHESES not in features
487         # the opening bracket is an optional paren
488         and opening_bracket.type == token.LPAR
489         and not opening_bracket.value
490         # the closing bracket is an optional paren
491         and closing_bracket.type == token.RPAR
492         and not closing_bracket.value
493         # it's not an import (optional parens are the only thing we can split on
494         # in this case; attempting a split without them is a waste of time)
495         and not line.is_import
496         # there are no standalone comments in the body
497         and not body.contains_standalone_comments(0)
498         # and we can actually remove the parens
499         and can_omit_invisible_parens(body, line_length, omit_on_explode=omit)
500     ):
501         omit = {id(closing_bracket), *omit}
502         try:
503             yield from right_hand_split(line, line_length, features=features, omit=omit)
504             return
505
506         except CannotSplit as e:
507             if not (
508                 can_be_split(body)
509                 or is_line_short_enough(body, line_length=line_length)
510             ):
511                 raise CannotSplit(
512                     "Splitting failed, body is still too long and can't be split."
513                 ) from e
514
515             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
516                 raise CannotSplit(
517                     "The current optional pair of parentheses is bound to fail to"
518                     " satisfy the splitting algorithm because the head or the tail"
519                     " contains multiline strings which by definition never fit one"
520                     " line."
521                 ) from e
522
523     ensure_visible(opening_bracket)
524     ensure_visible(closing_bracket)
525     for result in (head, body, tail):
526         if result:
527             yield result
528
529
530 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
531     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
532
533     Do nothing otherwise.
534
535     A left- or right-hand split is based on a pair of brackets. Content before
536     (and including) the opening bracket is left on one line, content inside the
537     brackets is put on a separate line, and finally content starting with and
538     following the closing bracket is put on a separate line.
539
540     Those are called `head`, `body`, and `tail`, respectively. If the split
541     produced the same line (all content in `head`) or ended up with an empty `body`
542     and the `tail` is just the closing bracket, then it's considered failed.
543     """
544     tail_len = len(str(tail).strip())
545     if not body:
546         if tail_len == 0:
547             raise CannotSplit("Splitting brackets produced the same line")
548
549         elif tail_len < 3:
550             raise CannotSplit(
551                 f"Splitting brackets on an empty body to save {tail_len} characters is"
552                 " not worth it"
553             )
554
555
556 def bracket_split_build_line(
557     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
558 ) -> Line:
559     """Return a new line with given `leaves` and respective comments from `original`.
560
561     If `is_body` is True, the result line is one-indented inside brackets and as such
562     has its first leaf's prefix normalized and a trailing comma added when expected.
563     """
564     result = Line(mode=original.mode, depth=original.depth)
565     if is_body:
566         result.inside_brackets = True
567         result.depth += 1
568         if leaves:
569             # Since body is a new indent level, remove spurious leading whitespace.
570             normalize_prefix(leaves[0], inside_brackets=True)
571             # Ensure a trailing comma for imports and standalone function arguments, but
572             # be careful not to add one after any comments or within type annotations.
573             no_commas = (
574                 original.is_def
575                 and opening_bracket.value == "("
576                 and not any(leaf.type == token.COMMA for leaf in leaves)
577                 # In particular, don't add one within a parenthesized return annotation.
578                 # Unfortunately the indicator we're in a return annotation (RARROW) may
579                 # be defined directly in the parent node, the parent of the parent ...
580                 # and so on depending on how complex the return annotation is.
581                 # This isn't perfect and there's some false negatives but they are in
582                 # contexts were a comma is actually fine.
583                 and not any(
584                     node.prev_sibling.type == RARROW
585                     for node in (
586                         leaves[0].parent,
587                         getattr(leaves[0].parent, "parent", None),
588                     )
589                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
590                 )
591             )
592
593             if original.is_import or no_commas:
594                 for i in range(len(leaves) - 1, -1, -1):
595                     if leaves[i].type == STANDALONE_COMMENT:
596                         continue
597
598                     if leaves[i].type != token.COMMA:
599                         new_comma = Leaf(token.COMMA, ",")
600                         leaves.insert(i + 1, new_comma)
601                     break
602
603     # Populate the line
604     for leaf in leaves:
605         result.append(leaf, preformatted=True)
606         for comment_after in original.comments_after(leaf):
607             result.append(comment_after, preformatted=True)
608     if is_body and should_split_line(result, opening_bracket):
609         result.should_split_rhs = True
610     return result
611
612
613 def dont_increase_indentation(split_func: Transformer) -> Transformer:
614     """Normalize prefix of the first leaf in every line returned by `split_func`.
615
616     This is a decorator over relevant split functions.
617     """
618
619     @wraps(split_func)
620     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
621         for line in split_func(line, features):
622             normalize_prefix(line.leaves[0], inside_brackets=True)
623             yield line
624
625     return split_wrapper
626
627
628 @dont_increase_indentation
629 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
630     """Split according to delimiters of the highest priority.
631
632     If the appropriate Features are given, the split will add trailing commas
633     also in function signatures and calls that contain `*` and `**`.
634     """
635     try:
636         last_leaf = line.leaves[-1]
637     except IndexError:
638         raise CannotSplit("Line empty") from None
639
640     bt = line.bracket_tracker
641     try:
642         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
643     except ValueError:
644         raise CannotSplit("No delimiters found") from None
645
646     if delimiter_priority == DOT_PRIORITY:
647         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
648             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
649
650     current_line = Line(
651         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
652     )
653     lowest_depth = sys.maxsize
654     trailing_comma_safe = True
655
656     def append_to_line(leaf: Leaf) -> Iterator[Line]:
657         """Append `leaf` to current line or to new line if appending impossible."""
658         nonlocal current_line
659         try:
660             current_line.append_safe(leaf, preformatted=True)
661         except ValueError:
662             yield current_line
663
664             current_line = Line(
665                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
666             )
667             current_line.append(leaf)
668
669     for leaf in line.leaves:
670         yield from append_to_line(leaf)
671
672         for comment_after in line.comments_after(leaf):
673             yield from append_to_line(comment_after)
674
675         lowest_depth = min(lowest_depth, leaf.bracket_depth)
676         if leaf.bracket_depth == lowest_depth:
677             if is_vararg(leaf, within={syms.typedargslist}):
678                 trailing_comma_safe = (
679                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
680                 )
681             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
682                 trailing_comma_safe = (
683                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
684                 )
685
686         leaf_priority = bt.delimiters.get(id(leaf))
687         if leaf_priority == delimiter_priority:
688             yield current_line
689
690             current_line = Line(
691                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
692             )
693     if current_line:
694         if (
695             trailing_comma_safe
696             and delimiter_priority == COMMA_PRIORITY
697             and current_line.leaves[-1].type != token.COMMA
698             and current_line.leaves[-1].type != STANDALONE_COMMENT
699         ):
700             new_comma = Leaf(token.COMMA, ",")
701             current_line.append(new_comma)
702         yield current_line
703
704
705 @dont_increase_indentation
706 def standalone_comment_split(
707     line: Line, features: Collection[Feature] = ()
708 ) -> Iterator[Line]:
709     """Split standalone comments from the rest of the line."""
710     if not line.contains_standalone_comments(0):
711         raise CannotSplit("Line does not have any standalone comments")
712
713     current_line = Line(
714         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
715     )
716
717     def append_to_line(leaf: Leaf) -> Iterator[Line]:
718         """Append `leaf` to current line or to new line if appending impossible."""
719         nonlocal current_line
720         try:
721             current_line.append_safe(leaf, preformatted=True)
722         except ValueError:
723             yield current_line
724
725             current_line = Line(
726                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
727             )
728             current_line.append(leaf)
729
730     for leaf in line.leaves:
731         yield from append_to_line(leaf)
732
733         for comment_after in line.comments_after(leaf):
734             yield from append_to_line(comment_after)
735
736     if current_line:
737         yield current_line
738
739
740 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
741     """Leave existing extra newlines if not `inside_brackets`. Remove everything
742     else.
743
744     Note: don't use backslashes for formatting or you'll lose your voting rights.
745     """
746     if not inside_brackets:
747         spl = leaf.prefix.split("#")
748         if "\\" not in spl[0]:
749             nl_count = spl[-1].count("\n")
750             if len(spl) > 1:
751                 nl_count -= 1
752             leaf.prefix = "\n" * nl_count
753             return
754
755     leaf.prefix = ""
756
757
758 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
759     """Make existing optional parentheses invisible or create new ones.
760
761     `parens_after` is a set of string leaf values immediately after which parens
762     should be put.
763
764     Standardizes on visible parentheses for single-element tuples, and keeps
765     existing visible parentheses for other tuples and generator expressions.
766     """
767     for pc in list_comments(node.prefix, is_endmarker=False):
768         if pc.value in FMT_OFF:
769             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
770             return
771     check_lpar = False
772     for index, child in enumerate(list(node.children)):
773         # Fixes a bug where invisible parens are not properly stripped from
774         # assignment statements that contain type annotations.
775         if isinstance(child, Node) and child.type == syms.annassign:
776             normalize_invisible_parens(child, parens_after=parens_after)
777
778         # Add parentheses around long tuple unpacking in assignments.
779         if (
780             index == 0
781             and isinstance(child, Node)
782             and child.type == syms.testlist_star_expr
783         ):
784             check_lpar = True
785
786         if check_lpar:
787             if child.type == syms.atom:
788                 if maybe_make_parens_invisible_in_atom(child, parent=node):
789                     wrap_in_parentheses(node, child, visible=False)
790             elif is_one_tuple(child):
791                 wrap_in_parentheses(node, child, visible=True)
792             elif node.type == syms.import_from:
793                 # "import from" nodes store parentheses directly as part of
794                 # the statement
795                 if child.type == token.LPAR:
796                     # make parentheses invisible
797                     child.value = ""  # type: ignore
798                     node.children[-1].value = ""  # type: ignore
799                 elif child.type != token.STAR:
800                     # insert invisible parentheses
801                     node.insert_child(index, Leaf(token.LPAR, ""))
802                     node.append_child(Leaf(token.RPAR, ""))
803                 break
804
805             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
806                 wrap_in_parentheses(node, child, visible=False)
807
808         check_lpar = isinstance(child, Leaf) and child.value in parens_after
809
810
811 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
812     """If it's safe, make the parens in the atom `node` invisible, recursively.
813     Additionally, remove repeated, adjacent invisible parens from the atom `node`
814     as they are redundant.
815
816     Returns whether the node should itself be wrapped in invisible parentheses.
817
818     """
819
820     if (
821         node.type != syms.atom
822         or is_empty_tuple(node)
823         or is_one_tuple(node)
824         or (is_yield(node) and parent.type != syms.expr_stmt)
825         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
826     ):
827         return False
828
829     if is_walrus_assignment(node):
830         if parent.type in [
831             syms.annassign,
832             syms.expr_stmt,
833             syms.assert_stmt,
834             syms.return_stmt,
835             # these ones aren't useful to end users, but they do please fuzzers
836             syms.for_stmt,
837             syms.del_stmt,
838         ]:
839             return False
840
841     first = node.children[0]
842     last = node.children[-1]
843     if first.type == token.LPAR and last.type == token.RPAR:
844         middle = node.children[1]
845         # make parentheses invisible
846         first.value = ""  # type: ignore
847         last.value = ""  # type: ignore
848         maybe_make_parens_invisible_in_atom(middle, parent=parent)
849
850         if is_atom_with_invisible_parens(middle):
851             # Strip the invisible parens from `middle` by replacing
852             # it with the child in-between the invisible parens
853             middle.replace(middle.children[1])
854
855         return False
856
857     return True
858
859
860 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
861     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
862
863     if not (opening_bracket.parent and opening_bracket.value in "[{("):
864         return False
865
866     # We're essentially checking if the body is delimited by commas and there's more
867     # than one of them (we're excluding the trailing comma and if the delimiter priority
868     # is still commas, that means there's more).
869     exclude = set()
870     trailing_comma = False
871     try:
872         last_leaf = line.leaves[-1]
873         if last_leaf.type == token.COMMA:
874             trailing_comma = True
875             exclude.add(id(last_leaf))
876         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
877     except (IndexError, ValueError):
878         return False
879
880     return max_priority == COMMA_PRIORITY and (
881         (line.mode.magic_trailing_comma and trailing_comma)
882         # always explode imports
883         or opening_bracket.parent.type in {syms.atom, syms.import_from}
884     )
885
886
887 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
888     """Generate sets of closing bracket IDs that should be omitted in a RHS.
889
890     Brackets can be omitted if the entire trailer up to and including
891     a preceding closing bracket fits in one line.
892
893     Yielded sets are cumulative (contain results of previous yields, too).  First
894     set is empty, unless the line should explode, in which case bracket pairs until
895     the one that needs to explode are omitted.
896     """
897
898     omit: Set[LeafID] = set()
899     if not line.magic_trailing_comma:
900         yield omit
901
902     length = 4 * line.depth
903     opening_bracket: Optional[Leaf] = None
904     closing_bracket: Optional[Leaf] = None
905     inner_brackets: Set[LeafID] = set()
906     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
907         length += leaf_length
908         if length > line_length:
909             break
910
911         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
912         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
913             break
914
915         if opening_bracket:
916             if leaf is opening_bracket:
917                 opening_bracket = None
918             elif leaf.type in CLOSING_BRACKETS:
919                 prev = line.leaves[index - 1] if index > 0 else None
920                 if (
921                     prev
922                     and prev.type == token.COMMA
923                     and not is_one_tuple_between(
924                         leaf.opening_bracket, leaf, line.leaves
925                     )
926                 ):
927                     # Never omit bracket pairs with trailing commas.
928                     # We need to explode on those.
929                     break
930
931                 inner_brackets.add(id(leaf))
932         elif leaf.type in CLOSING_BRACKETS:
933             prev = line.leaves[index - 1] if index > 0 else None
934             if prev and prev.type in OPENING_BRACKETS:
935                 # Empty brackets would fail a split so treat them as "inner"
936                 # brackets (e.g. only add them to the `omit` set if another
937                 # pair of brackets was good enough.
938                 inner_brackets.add(id(leaf))
939                 continue
940
941             if closing_bracket:
942                 omit.add(id(closing_bracket))
943                 omit.update(inner_brackets)
944                 inner_brackets.clear()
945                 yield omit
946
947             if (
948                 prev
949                 and prev.type == token.COMMA
950                 and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves)
951             ):
952                 # Never omit bracket pairs with trailing commas.
953                 # We need to explode on those.
954                 break
955
956             if leaf.value:
957                 opening_bracket = leaf.opening_bracket
958                 closing_bracket = leaf
959
960
961 def run_transformer(
962     line: Line,
963     transform: Transformer,
964     mode: Mode,
965     features: Collection[Feature],
966     *,
967     line_str: str = "",
968 ) -> List[Line]:
969     if not line_str:
970         line_str = line_to_string(line)
971     result: List[Line] = []
972     for transformed_line in transform(line, features):
973         if str(transformed_line).strip("\n") == line_str:
974             raise CannotTransform("Line transformer returned an unchanged result")
975
976         result.extend(transform_line(transformed_line, mode=mode, features=features))
977
978     if (
979         transform.__name__ != "rhs"
980         or not line.bracket_tracker.invisible
981         or any(bracket.value for bracket in line.bracket_tracker.invisible)
982         or line.contains_multiline_strings()
983         or result[0].contains_uncollapsable_type_comments()
984         or result[0].contains_unsplittable_type_ignore()
985         or is_line_short_enough(result[0], line_length=mode.line_length)
986         # If any leaves have no parents (which _can_ occur since
987         # `transform(line)` potentially destroys the line's underlying node
988         # structure), then we can't proceed. Doing so would cause the below
989         # call to `append_leaves()` to fail.
990         or any(leaf.parent is None for leaf in line.leaves)
991     ):
992         return result
993
994     line_copy = line.clone()
995     append_leaves(line_copy, line, line.leaves)
996     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
997     second_opinion = run_transformer(
998         line_copy, transform, mode, features_fop, line_str=line_str
999     )
1000     if all(
1001         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1002     ):
1003         result = second_opinion
1004     return result