]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Switch to Furo (#2793)
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 from functools import partial, wraps
5 import sys
6 from typing import Collection, Iterator, List, Optional, Set, Union
7
8 from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT
9 from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS
10 from black.nodes import Visitor, syms, first_child_is_arith, ensure_visible
11 from black.nodes import is_docstring, is_empty_tuple, is_one_tuple, is_one_tuple_between
12 from black.nodes import is_name_token, is_lpar_token, is_rpar_token
13 from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string
14 from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens
15 from black.nodes import wrap_in_parentheses
16 from black.brackets import max_delimiter_priority_in_atom
17 from black.brackets import DOT_PRIORITY, COMMA_PRIORITY
18 from black.lines import Line, line_to_string, is_line_short_enough
19 from black.lines import can_omit_invisible_parens, can_be_split, append_leaves
20 from black.comments import generate_comments, list_comments, FMT_OFF
21 from black.numerics import normalize_numeric_literal
22 from black.strings import get_string_prefix, fix_docstring
23 from black.strings import normalize_string_prefix, normalize_string_quotes
24 from black.trans import Transformer, CannotTransform, StringMerger
25 from black.trans import StringSplitter, StringParenWrapper, StringParenStripper
26 from black.mode import Mode, Feature, Preview
27
28 from blib2to3.pytree import Node, Leaf
29 from blib2to3.pgen2 import token
30
31
32 # types
33 LeafID = int
34 LN = Union[Leaf, Node]
35
36
37 class CannotSplit(CannotTransform):
38     """A readable split that fits the allotted line length is impossible."""
39
40
41 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
42 # See also https://github.com/mypyc/mypyc/issues/827.
43 class LineGenerator(Visitor[Line]):
44     """Generates reformatted Line objects.  Empty lines are not emitted.
45
46     Note: destroys the tree it's visiting by mutating prefixes of its leaves
47     in ways that will no longer stringify to valid Python code on the tree.
48     """
49
50     def __init__(self, mode: Mode) -> None:
51         self.mode = mode
52         self.current_line: Line
53         self.__post_init__()
54
55     def line(self, indent: int = 0) -> Iterator[Line]:
56         """Generate a line.
57
58         If the line is empty, only emit if it makes sense.
59         If the line is too long, split it first and then generate.
60
61         If any lines were generated, set up a new current_line.
62         """
63         if not self.current_line:
64             self.current_line.depth += indent
65             return  # Line is empty, don't emit. Creating a new one unnecessary.
66
67         complete_line = self.current_line
68         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
69         yield complete_line
70
71     def visit_default(self, node: LN) -> Iterator[Line]:
72         """Default `visit_*()` implementation. Recurses to children of `node`."""
73         if isinstance(node, Leaf):
74             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
75             for comment in generate_comments(node):
76                 if any_open_brackets:
77                     # any comment within brackets is subject to splitting
78                     self.current_line.append(comment)
79                 elif comment.type == token.COMMENT:
80                     # regular trailing comment
81                     self.current_line.append(comment)
82                     yield from self.line()
83
84                 else:
85                     # regular standalone comment
86                     yield from self.line()
87
88                     self.current_line.append(comment)
89                     yield from self.line()
90
91             normalize_prefix(node, inside_brackets=any_open_brackets)
92             if self.mode.string_normalization and node.type == token.STRING:
93                 node.value = normalize_string_prefix(node.value)
94                 node.value = normalize_string_quotes(node.value)
95             if node.type == token.NUMBER:
96                 normalize_numeric_literal(node)
97             if node.type not in WHITESPACE:
98                 self.current_line.append(node)
99         yield from super().visit_default(node)
100
101     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
102         """Increase indentation level, maybe yield a line."""
103         # In blib2to3 INDENT never holds comments.
104         yield from self.line(+1)
105         yield from self.visit_default(node)
106
107     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
108         """Decrease indentation level, maybe yield a line."""
109         # The current line might still wait for trailing comments.  At DEDENT time
110         # there won't be any (they would be prefixes on the preceding NEWLINE).
111         # Emit the line then.
112         yield from self.line()
113
114         # While DEDENT has no value, its prefix may contain standalone comments
115         # that belong to the current indentation level.  Get 'em.
116         yield from self.visit_default(node)
117
118         # Finally, emit the dedent.
119         yield from self.line(-1)
120
121     def visit_stmt(
122         self, node: Node, keywords: Set[str], parens: Set[str]
123     ) -> Iterator[Line]:
124         """Visit a statement.
125
126         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
127         `def`, `with`, `class`, `assert`, and assignments.
128
129         The relevant Python language `keywords` for a given statement will be
130         NAME leaves within it. This methods puts those on a separate line.
131
132         `parens` holds a set of string leaf values immediately after which
133         invisible parens should be put.
134         """
135         normalize_invisible_parens(node, parens_after=parens)
136         for child in node.children:
137             if is_name_token(child) and child.value in keywords:
138                 yield from self.line()
139
140             yield from self.visit(child)
141
142     def visit_match_case(self, node: Node) -> Iterator[Line]:
143         """Visit either a match or case statement."""
144         normalize_invisible_parens(node, parens_after=set())
145
146         yield from self.line()
147         for child in node.children:
148             yield from self.visit(child)
149
150     def visit_suite(self, node: Node) -> Iterator[Line]:
151         """Visit a suite."""
152         if self.mode.is_pyi and is_stub_suite(node):
153             yield from self.visit(node.children[2])
154         else:
155             yield from self.visit_default(node)
156
157     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
158         """Visit a statement without nested statements."""
159         if first_child_is_arith(node):
160             wrap_in_parentheses(node, node.children[0], visible=False)
161         is_suite_like = node.parent and node.parent.type in STATEMENT
162         if is_suite_like:
163             if self.mode.is_pyi and is_stub_body(node):
164                 yield from self.visit_default(node)
165             else:
166                 yield from self.line(+1)
167                 yield from self.visit_default(node)
168                 yield from self.line(-1)
169
170         else:
171             if (
172                 not self.mode.is_pyi
173                 or not node.parent
174                 or not is_stub_suite(node.parent)
175             ):
176                 yield from self.line()
177             yield from self.visit_default(node)
178
179     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
180         """Visit `async def`, `async for`, `async with`."""
181         yield from self.line()
182
183         children = iter(node.children)
184         for child in children:
185             yield from self.visit(child)
186
187             if child.type == token.ASYNC:
188                 break
189
190         internal_stmt = next(children)
191         for child in internal_stmt.children:
192             yield from self.visit(child)
193
194     def visit_decorators(self, node: Node) -> Iterator[Line]:
195         """Visit decorators."""
196         for child in node.children:
197             yield from self.line()
198             yield from self.visit(child)
199
200     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
201         """Remove a semicolon and put the other statement on a separate line."""
202         yield from self.line()
203
204     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
205         """End of file. Process outstanding comments and end with a newline."""
206         yield from self.visit_default(leaf)
207         yield from self.line()
208
209     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
210         if not self.current_line.bracket_tracker.any_open_brackets():
211             yield from self.line()
212         yield from self.visit_default(leaf)
213
214     def visit_factor(self, node: Node) -> Iterator[Line]:
215         """Force parentheses between a unary op and a binary power:
216
217         -2 ** 8 -> -(2 ** 8)
218         """
219         _operator, operand = node.children
220         if (
221             operand.type == syms.power
222             and len(operand.children) == 3
223             and operand.children[1].type == token.DOUBLESTAR
224         ):
225             lpar = Leaf(token.LPAR, "(")
226             rpar = Leaf(token.RPAR, ")")
227             index = operand.remove() or 0
228             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
229         yield from self.visit_default(node)
230
231     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
232         if is_docstring(leaf) and "\\\n" not in leaf.value:
233             # We're ignoring docstrings with backslash newline escapes because changing
234             # indentation of those changes the AST representation of the code.
235             docstring = normalize_string_prefix(leaf.value)
236             prefix = get_string_prefix(docstring)
237             docstring = docstring[len(prefix) :]  # Remove the prefix
238             quote_char = docstring[0]
239             # A natural way to remove the outer quotes is to do:
240             #   docstring = docstring.strip(quote_char)
241             # but that breaks on """""x""" (which is '""x').
242             # So we actually need to remove the first character and the next two
243             # characters but only if they are the same as the first.
244             quote_len = 1 if docstring[1] != quote_char else 3
245             docstring = docstring[quote_len:-quote_len]
246             docstring_started_empty = not docstring
247
248             if is_multiline_string(leaf):
249                 indent = " " * 4 * self.current_line.depth
250                 docstring = fix_docstring(docstring, indent)
251             else:
252                 docstring = docstring.strip()
253
254             if docstring:
255                 # Add some padding if the docstring starts / ends with a quote mark.
256                 if docstring[0] == quote_char:
257                     docstring = " " + docstring
258                 if docstring[-1] == quote_char:
259                     docstring += " "
260                 if docstring[-1] == "\\":
261                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
262                     if backslash_count % 2:
263                         # Odd number of tailing backslashes, add some padding to
264                         # avoid escaping the closing string quote.
265                         docstring += " "
266             elif not docstring_started_empty:
267                 docstring = " "
268
269             # We could enforce triple quotes at this point.
270             quote = quote_char * quote_len
271             leaf.value = prefix + quote + docstring + quote
272
273         yield from self.visit_default(leaf)
274
275     def __post_init__(self) -> None:
276         """You are in a twisty little maze of passages."""
277         self.current_line = Line(mode=self.mode)
278
279         v = self.visit_stmt
280         Ø: Set[str] = set()
281         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
282         self.visit_if_stmt = partial(
283             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
284         )
285         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
286         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
287         self.visit_try_stmt = partial(
288             v, keywords={"try", "except", "else", "finally"}, parens=Ø
289         )
290         self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
291         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
292         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
293         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
294         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
295         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
296         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
297         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
298         self.visit_async_funcdef = self.visit_async_stmt
299         self.visit_decorated = self.visit_decorators
300
301         # PEP 634
302         self.visit_match_stmt = self.visit_match_case
303         self.visit_case_block = self.visit_match_case
304
305
306 def transform_line(
307     line: Line, mode: Mode, features: Collection[Feature] = ()
308 ) -> Iterator[Line]:
309     """Transform a `line`, potentially splitting it into many lines.
310
311     They should fit in the allotted `line_length` but might not be able to.
312
313     `features` are syntactical features that may be used in the output.
314     """
315     if line.is_comment:
316         yield line
317         return
318
319     line_str = line_to_string(line)
320
321     ll = mode.line_length
322     sn = mode.string_normalization
323     string_merge = StringMerger(ll, sn)
324     string_paren_strip = StringParenStripper(ll, sn)
325     string_split = StringSplitter(ll, sn)
326     string_paren_wrap = StringParenWrapper(ll, sn)
327
328     transformers: List[Transformer]
329     if (
330         not line.contains_uncollapsable_type_comments()
331         and not line.should_split_rhs
332         and not line.magic_trailing_comma
333         and (
334             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
335             or line.contains_unsplittable_type_ignore()
336         )
337         and not (line.inside_brackets and line.contains_standalone_comments())
338     ):
339         # Only apply basic string preprocessing, since lines shouldn't be split here.
340         if Preview.string_processing in mode:
341             transformers = [string_merge, string_paren_strip]
342         else:
343             transformers = []
344     elif line.is_def:
345         transformers = [left_hand_split]
346     else:
347
348         def _rhs(
349             self: object, line: Line, features: Collection[Feature]
350         ) -> Iterator[Line]:
351             """Wraps calls to `right_hand_split`.
352
353             The calls increasingly `omit` right-hand trailers (bracket pairs with
354             content), meaning the trailers get glued together to split on another
355             bracket pair instead.
356             """
357             for omit in generate_trailers_to_omit(line, mode.line_length):
358                 lines = list(
359                     right_hand_split(line, mode.line_length, features, omit=omit)
360                 )
361                 # Note: this check is only able to figure out if the first line of the
362                 # *current* transformation fits in the line length.  This is true only
363                 # for simple cases.  All others require running more transforms via
364                 # `transform_line()`.  This check doesn't know if those would succeed.
365                 if is_line_short_enough(lines[0], line_length=mode.line_length):
366                     yield from lines
367                     return
368
369             # All splits failed, best effort split with no omits.
370             # This mostly happens to multiline strings that are by definition
371             # reported as not fitting a single line, as well as lines that contain
372             # trailing commas (those have to be exploded).
373             yield from right_hand_split(
374                 line, line_length=mode.line_length, features=features
375             )
376
377         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
378         # __name__ attribute which is needed in `run_transformer` further down.
379         # Unfortunately a nested class breaks mypyc too. So a class must be created
380         # via type ... https://github.com/mypyc/mypyc/issues/884
381         rhs = type("rhs", (), {"__call__": _rhs})()
382
383         if Preview.string_processing in mode:
384             if line.inside_brackets:
385                 transformers = [
386                     string_merge,
387                     string_paren_strip,
388                     string_split,
389                     delimiter_split,
390                     standalone_comment_split,
391                     string_paren_wrap,
392                     rhs,
393                 ]
394             else:
395                 transformers = [
396                     string_merge,
397                     string_paren_strip,
398                     string_split,
399                     string_paren_wrap,
400                     rhs,
401                 ]
402         else:
403             if line.inside_brackets:
404                 transformers = [delimiter_split, standalone_comment_split, rhs]
405             else:
406                 transformers = [rhs]
407
408     for transform in transformers:
409         # We are accumulating lines in `result` because we might want to abort
410         # mission and return the original line in the end, or attempt a different
411         # split altogether.
412         try:
413             result = run_transformer(line, transform, mode, features, line_str=line_str)
414         except CannotTransform:
415             continue
416         else:
417             yield from result
418             break
419
420     else:
421         yield line
422
423
424 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
425     """Split line into many lines, starting with the first matching bracket pair.
426
427     Note: this usually looks weird, only use this for function definitions.
428     Prefer RHS otherwise.  This is why this function is not symmetrical with
429     :func:`right_hand_split` which also handles optional parentheses.
430     """
431     tail_leaves: List[Leaf] = []
432     body_leaves: List[Leaf] = []
433     head_leaves: List[Leaf] = []
434     current_leaves = head_leaves
435     matching_bracket: Optional[Leaf] = None
436     for leaf in line.leaves:
437         if (
438             current_leaves is body_leaves
439             and leaf.type in CLOSING_BRACKETS
440             and leaf.opening_bracket is matching_bracket
441         ):
442             current_leaves = tail_leaves if body_leaves else head_leaves
443         current_leaves.append(leaf)
444         if current_leaves is head_leaves:
445             if leaf.type in OPENING_BRACKETS:
446                 matching_bracket = leaf
447                 current_leaves = body_leaves
448     if not matching_bracket:
449         raise CannotSplit("No brackets found")
450
451     head = bracket_split_build_line(head_leaves, line, matching_bracket)
452     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
453     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
454     bracket_split_succeeded_or_raise(head, body, tail)
455     for result in (head, body, tail):
456         if result:
457             yield result
458
459
460 def right_hand_split(
461     line: Line,
462     line_length: int,
463     features: Collection[Feature] = (),
464     omit: Collection[LeafID] = (),
465 ) -> Iterator[Line]:
466     """Split line into many lines, starting with the last matching bracket pair.
467
468     If the split was by optional parentheses, attempt splitting without them, too.
469     `omit` is a collection of closing bracket IDs that shouldn't be considered for
470     this split.
471
472     Note: running this function modifies `bracket_depth` on the leaves of `line`.
473     """
474     tail_leaves: List[Leaf] = []
475     body_leaves: List[Leaf] = []
476     head_leaves: List[Leaf] = []
477     current_leaves = tail_leaves
478     opening_bracket: Optional[Leaf] = None
479     closing_bracket: Optional[Leaf] = None
480     for leaf in reversed(line.leaves):
481         if current_leaves is body_leaves:
482             if leaf is opening_bracket:
483                 current_leaves = head_leaves if body_leaves else tail_leaves
484         current_leaves.append(leaf)
485         if current_leaves is tail_leaves:
486             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
487                 opening_bracket = leaf.opening_bracket
488                 closing_bracket = leaf
489                 current_leaves = body_leaves
490     if not (opening_bracket and closing_bracket and head_leaves):
491         # If there is no opening or closing_bracket that means the split failed and
492         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
493         # the matching `opening_bracket` wasn't available on `line` anymore.
494         raise CannotSplit("No brackets found")
495
496     tail_leaves.reverse()
497     body_leaves.reverse()
498     head_leaves.reverse()
499     head = bracket_split_build_line(head_leaves, line, opening_bracket)
500     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
501     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
502     bracket_split_succeeded_or_raise(head, body, tail)
503     if (
504         Feature.FORCE_OPTIONAL_PARENTHESES not in features
505         # the opening bracket is an optional paren
506         and opening_bracket.type == token.LPAR
507         and not opening_bracket.value
508         # the closing bracket is an optional paren
509         and closing_bracket.type == token.RPAR
510         and not closing_bracket.value
511         # it's not an import (optional parens are the only thing we can split on
512         # in this case; attempting a split without them is a waste of time)
513         and not line.is_import
514         # there are no standalone comments in the body
515         and not body.contains_standalone_comments(0)
516         # and we can actually remove the parens
517         and can_omit_invisible_parens(body, line_length, omit_on_explode=omit)
518     ):
519         omit = {id(closing_bracket), *omit}
520         try:
521             yield from right_hand_split(line, line_length, features=features, omit=omit)
522             return
523
524         except CannotSplit as e:
525             if not (
526                 can_be_split(body)
527                 or is_line_short_enough(body, line_length=line_length)
528             ):
529                 raise CannotSplit(
530                     "Splitting failed, body is still too long and can't be split."
531                 ) from e
532
533             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
534                 raise CannotSplit(
535                     "The current optional pair of parentheses is bound to fail to"
536                     " satisfy the splitting algorithm because the head or the tail"
537                     " contains multiline strings which by definition never fit one"
538                     " line."
539                 ) from e
540
541     ensure_visible(opening_bracket)
542     ensure_visible(closing_bracket)
543     for result in (head, body, tail):
544         if result:
545             yield result
546
547
548 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
549     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
550
551     Do nothing otherwise.
552
553     A left- or right-hand split is based on a pair of brackets. Content before
554     (and including) the opening bracket is left on one line, content inside the
555     brackets is put on a separate line, and finally content starting with and
556     following the closing bracket is put on a separate line.
557
558     Those are called `head`, `body`, and `tail`, respectively. If the split
559     produced the same line (all content in `head`) or ended up with an empty `body`
560     and the `tail` is just the closing bracket, then it's considered failed.
561     """
562     tail_len = len(str(tail).strip())
563     if not body:
564         if tail_len == 0:
565             raise CannotSplit("Splitting brackets produced the same line")
566
567         elif tail_len < 3:
568             raise CannotSplit(
569                 f"Splitting brackets on an empty body to save {tail_len} characters is"
570                 " not worth it"
571             )
572
573
574 def bracket_split_build_line(
575     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
576 ) -> Line:
577     """Return a new line with given `leaves` and respective comments from `original`.
578
579     If `is_body` is True, the result line is one-indented inside brackets and as such
580     has its first leaf's prefix normalized and a trailing comma added when expected.
581     """
582     result = Line(mode=original.mode, depth=original.depth)
583     if is_body:
584         result.inside_brackets = True
585         result.depth += 1
586         if leaves:
587             # Since body is a new indent level, remove spurious leading whitespace.
588             normalize_prefix(leaves[0], inside_brackets=True)
589             # Ensure a trailing comma for imports and standalone function arguments, but
590             # be careful not to add one after any comments or within type annotations.
591             no_commas = (
592                 original.is_def
593                 and opening_bracket.value == "("
594                 and not any(leaf.type == token.COMMA for leaf in leaves)
595                 # In particular, don't add one within a parenthesized return annotation.
596                 # Unfortunately the indicator we're in a return annotation (RARROW) may
597                 # be defined directly in the parent node, the parent of the parent ...
598                 # and so on depending on how complex the return annotation is.
599                 # This isn't perfect and there's some false negatives but they are in
600                 # contexts were a comma is actually fine.
601                 and not any(
602                     node.prev_sibling.type == RARROW
603                     for node in (
604                         leaves[0].parent,
605                         getattr(leaves[0].parent, "parent", None),
606                     )
607                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
608                 )
609             )
610
611             if original.is_import or no_commas:
612                 for i in range(len(leaves) - 1, -1, -1):
613                     if leaves[i].type == STANDALONE_COMMENT:
614                         continue
615
616                     if leaves[i].type != token.COMMA:
617                         new_comma = Leaf(token.COMMA, ",")
618                         leaves.insert(i + 1, new_comma)
619                     break
620
621     # Populate the line
622     for leaf in leaves:
623         result.append(leaf, preformatted=True)
624         for comment_after in original.comments_after(leaf):
625             result.append(comment_after, preformatted=True)
626     if is_body and should_split_line(result, opening_bracket):
627         result.should_split_rhs = True
628     return result
629
630
631 def dont_increase_indentation(split_func: Transformer) -> Transformer:
632     """Normalize prefix of the first leaf in every line returned by `split_func`.
633
634     This is a decorator over relevant split functions.
635     """
636
637     @wraps(split_func)
638     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
639         for line in split_func(line, features):
640             normalize_prefix(line.leaves[0], inside_brackets=True)
641             yield line
642
643     return split_wrapper
644
645
646 @dont_increase_indentation
647 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
648     """Split according to delimiters of the highest priority.
649
650     If the appropriate Features are given, the split will add trailing commas
651     also in function signatures and calls that contain `*` and `**`.
652     """
653     try:
654         last_leaf = line.leaves[-1]
655     except IndexError:
656         raise CannotSplit("Line empty") from None
657
658     bt = line.bracket_tracker
659     try:
660         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
661     except ValueError:
662         raise CannotSplit("No delimiters found") from None
663
664     if delimiter_priority == DOT_PRIORITY:
665         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
666             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
667
668     current_line = Line(
669         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
670     )
671     lowest_depth = sys.maxsize
672     trailing_comma_safe = True
673
674     def append_to_line(leaf: Leaf) -> Iterator[Line]:
675         """Append `leaf` to current line or to new line if appending impossible."""
676         nonlocal current_line
677         try:
678             current_line.append_safe(leaf, preformatted=True)
679         except ValueError:
680             yield current_line
681
682             current_line = Line(
683                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
684             )
685             current_line.append(leaf)
686
687     for leaf in line.leaves:
688         yield from append_to_line(leaf)
689
690         for comment_after in line.comments_after(leaf):
691             yield from append_to_line(comment_after)
692
693         lowest_depth = min(lowest_depth, leaf.bracket_depth)
694         if leaf.bracket_depth == lowest_depth:
695             if is_vararg(leaf, within={syms.typedargslist}):
696                 trailing_comma_safe = (
697                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
698                 )
699             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
700                 trailing_comma_safe = (
701                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
702                 )
703
704         leaf_priority = bt.delimiters.get(id(leaf))
705         if leaf_priority == delimiter_priority:
706             yield current_line
707
708             current_line = Line(
709                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
710             )
711     if current_line:
712         if (
713             trailing_comma_safe
714             and delimiter_priority == COMMA_PRIORITY
715             and current_line.leaves[-1].type != token.COMMA
716             and current_line.leaves[-1].type != STANDALONE_COMMENT
717         ):
718             new_comma = Leaf(token.COMMA, ",")
719             current_line.append(new_comma)
720         yield current_line
721
722
723 @dont_increase_indentation
724 def standalone_comment_split(
725     line: Line, features: Collection[Feature] = ()
726 ) -> Iterator[Line]:
727     """Split standalone comments from the rest of the line."""
728     if not line.contains_standalone_comments(0):
729         raise CannotSplit("Line does not have any standalone comments")
730
731     current_line = Line(
732         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
733     )
734
735     def append_to_line(leaf: Leaf) -> Iterator[Line]:
736         """Append `leaf` to current line or to new line if appending impossible."""
737         nonlocal current_line
738         try:
739             current_line.append_safe(leaf, preformatted=True)
740         except ValueError:
741             yield current_line
742
743             current_line = Line(
744                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
745             )
746             current_line.append(leaf)
747
748     for leaf in line.leaves:
749         yield from append_to_line(leaf)
750
751         for comment_after in line.comments_after(leaf):
752             yield from append_to_line(comment_after)
753
754     if current_line:
755         yield current_line
756
757
758 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
759     """Leave existing extra newlines if not `inside_brackets`. Remove everything
760     else.
761
762     Note: don't use backslashes for formatting or you'll lose your voting rights.
763     """
764     if not inside_brackets:
765         spl = leaf.prefix.split("#")
766         if "\\" not in spl[0]:
767             nl_count = spl[-1].count("\n")
768             if len(spl) > 1:
769                 nl_count -= 1
770             leaf.prefix = "\n" * nl_count
771             return
772
773     leaf.prefix = ""
774
775
776 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
777     """Make existing optional parentheses invisible or create new ones.
778
779     `parens_after` is a set of string leaf values immediately after which parens
780     should be put.
781
782     Standardizes on visible parentheses for single-element tuples, and keeps
783     existing visible parentheses for other tuples and generator expressions.
784     """
785     for pc in list_comments(node.prefix, is_endmarker=False):
786         if pc.value in FMT_OFF:
787             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
788             return
789     check_lpar = False
790     for index, child in enumerate(list(node.children)):
791         # Fixes a bug where invisible parens are not properly stripped from
792         # assignment statements that contain type annotations.
793         if isinstance(child, Node) and child.type == syms.annassign:
794             normalize_invisible_parens(child, parens_after=parens_after)
795
796         # Add parentheses around long tuple unpacking in assignments.
797         if (
798             index == 0
799             and isinstance(child, Node)
800             and child.type == syms.testlist_star_expr
801         ):
802             check_lpar = True
803
804         if check_lpar:
805             if child.type == syms.atom:
806                 if maybe_make_parens_invisible_in_atom(child, parent=node):
807                     wrap_in_parentheses(node, child, visible=False)
808             elif is_one_tuple(child):
809                 wrap_in_parentheses(node, child, visible=True)
810             elif node.type == syms.import_from:
811                 # "import from" nodes store parentheses directly as part of
812                 # the statement
813                 if is_lpar_token(child):
814                     assert is_rpar_token(node.children[-1])
815                     # make parentheses invisible
816                     child.value = ""
817                     node.children[-1].value = ""
818                 elif child.type != token.STAR:
819                     # insert invisible parentheses
820                     node.insert_child(index, Leaf(token.LPAR, ""))
821                     node.append_child(Leaf(token.RPAR, ""))
822                 break
823
824             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
825                 wrap_in_parentheses(node, child, visible=False)
826
827         check_lpar = isinstance(child, Leaf) and child.value in parens_after
828
829
830 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
831     """If it's safe, make the parens in the atom `node` invisible, recursively.
832     Additionally, remove repeated, adjacent invisible parens from the atom `node`
833     as they are redundant.
834
835     Returns whether the node should itself be wrapped in invisible parentheses.
836
837     """
838
839     if (
840         node.type != syms.atom
841         or is_empty_tuple(node)
842         or is_one_tuple(node)
843         or (is_yield(node) and parent.type != syms.expr_stmt)
844         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
845     ):
846         return False
847
848     if is_walrus_assignment(node):
849         if parent.type in [
850             syms.annassign,
851             syms.expr_stmt,
852             syms.assert_stmt,
853             syms.return_stmt,
854             # these ones aren't useful to end users, but they do please fuzzers
855             syms.for_stmt,
856             syms.del_stmt,
857         ]:
858             return False
859
860     first = node.children[0]
861     last = node.children[-1]
862     if is_lpar_token(first) and is_rpar_token(last):
863         middle = node.children[1]
864         # make parentheses invisible
865         first.value = ""
866         last.value = ""
867         maybe_make_parens_invisible_in_atom(middle, parent=parent)
868
869         if is_atom_with_invisible_parens(middle):
870             # Strip the invisible parens from `middle` by replacing
871             # it with the child in-between the invisible parens
872             middle.replace(middle.children[1])
873
874         return False
875
876     return True
877
878
879 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
880     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
881
882     if not (opening_bracket.parent and opening_bracket.value in "[{("):
883         return False
884
885     # We're essentially checking if the body is delimited by commas and there's more
886     # than one of them (we're excluding the trailing comma and if the delimiter priority
887     # is still commas, that means there's more).
888     exclude = set()
889     trailing_comma = False
890     try:
891         last_leaf = line.leaves[-1]
892         if last_leaf.type == token.COMMA:
893             trailing_comma = True
894             exclude.add(id(last_leaf))
895         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
896     except (IndexError, ValueError):
897         return False
898
899     return max_priority == COMMA_PRIORITY and (
900         (line.mode.magic_trailing_comma and trailing_comma)
901         # always explode imports
902         or opening_bracket.parent.type in {syms.atom, syms.import_from}
903     )
904
905
906 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
907     """Generate sets of closing bracket IDs that should be omitted in a RHS.
908
909     Brackets can be omitted if the entire trailer up to and including
910     a preceding closing bracket fits in one line.
911
912     Yielded sets are cumulative (contain results of previous yields, too).  First
913     set is empty, unless the line should explode, in which case bracket pairs until
914     the one that needs to explode are omitted.
915     """
916
917     omit: Set[LeafID] = set()
918     if not line.magic_trailing_comma:
919         yield omit
920
921     length = 4 * line.depth
922     opening_bracket: Optional[Leaf] = None
923     closing_bracket: Optional[Leaf] = None
924     inner_brackets: Set[LeafID] = set()
925     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
926         length += leaf_length
927         if length > line_length:
928             break
929
930         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
931         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
932             break
933
934         if opening_bracket:
935             if leaf is opening_bracket:
936                 opening_bracket = None
937             elif leaf.type in CLOSING_BRACKETS:
938                 prev = line.leaves[index - 1] if index > 0 else None
939                 if (
940                     prev
941                     and prev.type == token.COMMA
942                     and not is_one_tuple_between(
943                         leaf.opening_bracket, leaf, line.leaves
944                     )
945                 ):
946                     # Never omit bracket pairs with trailing commas.
947                     # We need to explode on those.
948                     break
949
950                 inner_brackets.add(id(leaf))
951         elif leaf.type in CLOSING_BRACKETS:
952             prev = line.leaves[index - 1] if index > 0 else None
953             if prev and prev.type in OPENING_BRACKETS:
954                 # Empty brackets would fail a split so treat them as "inner"
955                 # brackets (e.g. only add them to the `omit` set if another
956                 # pair of brackets was good enough.
957                 inner_brackets.add(id(leaf))
958                 continue
959
960             if closing_bracket:
961                 omit.add(id(closing_bracket))
962                 omit.update(inner_brackets)
963                 inner_brackets.clear()
964                 yield omit
965
966             if (
967                 prev
968                 and prev.type == token.COMMA
969                 and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves)
970             ):
971                 # Never omit bracket pairs with trailing commas.
972                 # We need to explode on those.
973                 break
974
975             if leaf.value:
976                 opening_bracket = leaf.opening_bracket
977                 closing_bracket = leaf
978
979
980 def run_transformer(
981     line: Line,
982     transform: Transformer,
983     mode: Mode,
984     features: Collection[Feature],
985     *,
986     line_str: str = "",
987 ) -> List[Line]:
988     if not line_str:
989         line_str = line_to_string(line)
990     result: List[Line] = []
991     for transformed_line in transform(line, features):
992         if str(transformed_line).strip("\n") == line_str:
993             raise CannotTransform("Line transformer returned an unchanged result")
994
995         result.extend(transform_line(transformed_line, mode=mode, features=features))
996
997     if (
998         transform.__class__.__name__ != "rhs"
999         or not line.bracket_tracker.invisible
1000         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1001         or line.contains_multiline_strings()
1002         or result[0].contains_uncollapsable_type_comments()
1003         or result[0].contains_unsplittable_type_ignore()
1004         or is_line_short_enough(result[0], line_length=mode.line_length)
1005         # If any leaves have no parents (which _can_ occur since
1006         # `transform(line)` potentially destroys the line's underlying node
1007         # structure), then we can't proceed. Doing so would cause the below
1008         # call to `append_leaves()` to fail.
1009         or any(leaf.parent is None for leaf in line.leaves)
1010     ):
1011         return result
1012
1013     line_copy = line.clone()
1014     append_leaves(line_copy, line, line.leaves)
1015     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1016     second_opinion = run_transformer(
1017         line_copy, transform, mode, features_fop, line_str=line_str
1018     )
1019     if all(
1020         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1021     ):
1022         result = second_opinion
1023     return result