]> git.madduck.net Git - etc/vim.git/blob - src/black/linegen.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Update email (#3235)
[etc/vim.git] / src / black / linegen.py
1 """
2 Generating lines of code.
3 """
4 import sys
5 from functools import partial, wraps
6 from typing import Collection, Iterator, List, Optional, Set, Union, cast
7
8 from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, max_delimiter_priority_in_atom
9 from black.comments import FMT_OFF, generate_comments, list_comments
10 from black.lines import (
11     Line,
12     append_leaves,
13     can_be_split,
14     can_omit_invisible_parens,
15     is_line_short_enough,
16     line_to_string,
17 )
18 from black.mode import Feature, Mode, Preview
19 from black.nodes import (
20     ASSIGNMENTS,
21     CLOSING_BRACKETS,
22     OPENING_BRACKETS,
23     RARROW,
24     STANDALONE_COMMENT,
25     STATEMENT,
26     WHITESPACE,
27     Visitor,
28     ensure_visible,
29     is_arith_like,
30     is_atom_with_invisible_parens,
31     is_docstring,
32     is_empty_tuple,
33     is_lpar_token,
34     is_multiline_string,
35     is_name_token,
36     is_one_sequence_between,
37     is_one_tuple,
38     is_rpar_token,
39     is_stub_body,
40     is_stub_suite,
41     is_vararg,
42     is_walrus_assignment,
43     is_yield,
44     syms,
45     wrap_in_parentheses,
46 )
47 from black.numerics import normalize_numeric_literal
48 from black.strings import (
49     fix_docstring,
50     get_string_prefix,
51     normalize_string_prefix,
52     normalize_string_quotes,
53 )
54 from black.trans import (
55     CannotTransform,
56     StringMerger,
57     StringParenStripper,
58     StringParenWrapper,
59     StringSplitter,
60     Transformer,
61     hug_power_op,
62 )
63 from blib2to3.pgen2 import token
64 from blib2to3.pytree import Leaf, Node
65
66 # types
67 LeafID = int
68 LN = Union[Leaf, Node]
69
70
71 class CannotSplit(CannotTransform):
72     """A readable split that fits the allotted line length is impossible."""
73
74
75 # This isn't a dataclass because @dataclass + Generic breaks mypyc.
76 # See also https://github.com/mypyc/mypyc/issues/827.
77 class LineGenerator(Visitor[Line]):
78     """Generates reformatted Line objects.  Empty lines are not emitted.
79
80     Note: destroys the tree it's visiting by mutating prefixes of its leaves
81     in ways that will no longer stringify to valid Python code on the tree.
82     """
83
84     def __init__(self, mode: Mode) -> None:
85         self.mode = mode
86         self.current_line: Line
87         self.__post_init__()
88
89     def line(self, indent: int = 0) -> Iterator[Line]:
90         """Generate a line.
91
92         If the line is empty, only emit if it makes sense.
93         If the line is too long, split it first and then generate.
94
95         If any lines were generated, set up a new current_line.
96         """
97         if not self.current_line:
98             self.current_line.depth += indent
99             return  # Line is empty, don't emit. Creating a new one unnecessary.
100
101         complete_line = self.current_line
102         self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)
103         yield complete_line
104
105     def visit_default(self, node: LN) -> Iterator[Line]:
106         """Default `visit_*()` implementation. Recurses to children of `node`."""
107         if isinstance(node, Leaf):
108             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
109             for comment in generate_comments(node, preview=self.mode.preview):
110                 if any_open_brackets:
111                     # any comment within brackets is subject to splitting
112                     self.current_line.append(comment)
113                 elif comment.type == token.COMMENT:
114                     # regular trailing comment
115                     self.current_line.append(comment)
116                     yield from self.line()
117
118                 else:
119                     # regular standalone comment
120                     yield from self.line()
121
122                     self.current_line.append(comment)
123                     yield from self.line()
124
125             normalize_prefix(node, inside_brackets=any_open_brackets)
126             if self.mode.string_normalization and node.type == token.STRING:
127                 node.value = normalize_string_prefix(node.value)
128                 node.value = normalize_string_quotes(node.value)
129             if node.type == token.NUMBER:
130                 normalize_numeric_literal(node)
131             if node.type not in WHITESPACE:
132                 self.current_line.append(node)
133         yield from super().visit_default(node)
134
135     def visit_INDENT(self, node: Leaf) -> Iterator[Line]:
136         """Increase indentation level, maybe yield a line."""
137         # In blib2to3 INDENT never holds comments.
138         yield from self.line(+1)
139         yield from self.visit_default(node)
140
141     def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:
142         """Decrease indentation level, maybe yield a line."""
143         # The current line might still wait for trailing comments.  At DEDENT time
144         # there won't be any (they would be prefixes on the preceding NEWLINE).
145         # Emit the line then.
146         yield from self.line()
147
148         # While DEDENT has no value, its prefix may contain standalone comments
149         # that belong to the current indentation level.  Get 'em.
150         yield from self.visit_default(node)
151
152         # Finally, emit the dedent.
153         yield from self.line(-1)
154
155     def visit_stmt(
156         self, node: Node, keywords: Set[str], parens: Set[str]
157     ) -> Iterator[Line]:
158         """Visit a statement.
159
160         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
161         `def`, `with`, `class`, `assert`, and assignments.
162
163         The relevant Python language `keywords` for a given statement will be
164         NAME leaves within it. This methods puts those on a separate line.
165
166         `parens` holds a set of string leaf values immediately after which
167         invisible parens should be put.
168         """
169         normalize_invisible_parens(node, parens_after=parens, preview=self.mode.preview)
170         for child in node.children:
171             if is_name_token(child) and child.value in keywords:
172                 yield from self.line()
173
174             yield from self.visit(child)
175
176     def visit_funcdef(self, node: Node) -> Iterator[Line]:
177         """Visit function definition."""
178         if Preview.annotation_parens not in self.mode:
179             yield from self.visit_stmt(node, keywords={"def"}, parens=set())
180         else:
181             yield from self.line()
182
183             # Remove redundant brackets around return type annotation.
184             is_return_annotation = False
185             for child in node.children:
186                 if child.type == token.RARROW:
187                     is_return_annotation = True
188                 elif is_return_annotation:
189                     if child.type == syms.atom and child.children[0].type == token.LPAR:
190                         if maybe_make_parens_invisible_in_atom(
191                             child,
192                             parent=node,
193                             remove_brackets_around_comma=False,
194                         ):
195                             wrap_in_parentheses(node, child, visible=False)
196                     else:
197                         wrap_in_parentheses(node, child, visible=False)
198                     is_return_annotation = False
199
200             for child in node.children:
201                 yield from self.visit(child)
202
203     def visit_match_case(self, node: Node) -> Iterator[Line]:
204         """Visit either a match or case statement."""
205         normalize_invisible_parens(node, parens_after=set(), preview=self.mode.preview)
206
207         yield from self.line()
208         for child in node.children:
209             yield from self.visit(child)
210
211     def visit_suite(self, node: Node) -> Iterator[Line]:
212         """Visit a suite."""
213         if self.mode.is_pyi and is_stub_suite(node):
214             yield from self.visit(node.children[2])
215         else:
216             yield from self.visit_default(node)
217
218     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
219         """Visit a statement without nested statements."""
220         prev_type: Optional[int] = None
221         for child in node.children:
222             if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):
223                 wrap_in_parentheses(node, child, visible=False)
224             prev_type = child.type
225
226         is_suite_like = node.parent and node.parent.type in STATEMENT
227         if is_suite_like:
228             if self.mode.is_pyi and is_stub_body(node):
229                 yield from self.visit_default(node)
230             else:
231                 yield from self.line(+1)
232                 yield from self.visit_default(node)
233                 yield from self.line(-1)
234
235         else:
236             if (
237                 not self.mode.is_pyi
238                 or not node.parent
239                 or not is_stub_suite(node.parent)
240             ):
241                 yield from self.line()
242             yield from self.visit_default(node)
243
244     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
245         """Visit `async def`, `async for`, `async with`."""
246         yield from self.line()
247
248         children = iter(node.children)
249         for child in children:
250             yield from self.visit(child)
251
252             if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:
253                 # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async
254                 # line.
255                 break
256
257         internal_stmt = next(children)
258         for child in internal_stmt.children:
259             yield from self.visit(child)
260
261     def visit_decorators(self, node: Node) -> Iterator[Line]:
262         """Visit decorators."""
263         for child in node.children:
264             yield from self.line()
265             yield from self.visit(child)
266
267     def visit_power(self, node: Node) -> Iterator[Line]:
268         for idx, leaf in enumerate(node.children[:-1]):
269             next_leaf = node.children[idx + 1]
270
271             if not isinstance(leaf, Leaf):
272                 continue
273
274             value = leaf.value.lower()
275             if (
276                 leaf.type == token.NUMBER
277                 and next_leaf.type == syms.trailer
278                 # Ensure that we are in an attribute trailer
279                 and next_leaf.children[0].type == token.DOT
280                 # It shouldn't wrap hexadecimal, binary and octal literals
281                 and not value.startswith(("0x", "0b", "0o"))
282                 # It shouldn't wrap complex literals
283                 and "j" not in value
284             ):
285                 wrap_in_parentheses(node, leaf)
286
287         if Preview.remove_redundant_parens in self.mode:
288             remove_await_parens(node)
289
290         yield from self.visit_default(node)
291
292     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
293         """Remove a semicolon and put the other statement on a separate line."""
294         yield from self.line()
295
296     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
297         """End of file. Process outstanding comments and end with a newline."""
298         yield from self.visit_default(leaf)
299         yield from self.line()
300
301     def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
302         if not self.current_line.bracket_tracker.any_open_brackets():
303             yield from self.line()
304         yield from self.visit_default(leaf)
305
306     def visit_factor(self, node: Node) -> Iterator[Line]:
307         """Force parentheses between a unary op and a binary power:
308
309         -2 ** 8 -> -(2 ** 8)
310         """
311         _operator, operand = node.children
312         if (
313             operand.type == syms.power
314             and len(operand.children) == 3
315             and operand.children[1].type == token.DOUBLESTAR
316         ):
317             lpar = Leaf(token.LPAR, "(")
318             rpar = Leaf(token.RPAR, ")")
319             index = operand.remove() or 0
320             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
321         yield from self.visit_default(node)
322
323     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
324         if is_docstring(leaf) and "\\\n" not in leaf.value:
325             # We're ignoring docstrings with backslash newline escapes because changing
326             # indentation of those changes the AST representation of the code.
327             if Preview.normalize_docstring_quotes_and_prefixes_properly in self.mode:
328                 # There was a bug where --skip-string-normalization wouldn't stop us
329                 # from normalizing docstring prefixes. To maintain stability, we can
330                 # only address this buggy behaviour while the preview style is enabled.
331                 if self.mode.string_normalization:
332                     docstring = normalize_string_prefix(leaf.value)
333                     # visit_default() does handle string normalization for us, but
334                     # since this method acts differently depending on quote style (ex.
335                     # see padding logic below), there's a possibility for unstable
336                     # formatting as visit_default() is called *after*. To avoid a
337                     # situation where this function formats a docstring differently on
338                     # the second pass, normalize it early.
339                     docstring = normalize_string_quotes(docstring)
340                 else:
341                     docstring = leaf.value
342             else:
343                 # ... otherwise, we'll keep the buggy behaviour >.<
344                 docstring = normalize_string_prefix(leaf.value)
345             prefix = get_string_prefix(docstring)
346             docstring = docstring[len(prefix) :]  # Remove the prefix
347             quote_char = docstring[0]
348             # A natural way to remove the outer quotes is to do:
349             #   docstring = docstring.strip(quote_char)
350             # but that breaks on """""x""" (which is '""x').
351             # So we actually need to remove the first character and the next two
352             # characters but only if they are the same as the first.
353             quote_len = 1 if docstring[1] != quote_char else 3
354             docstring = docstring[quote_len:-quote_len]
355             docstring_started_empty = not docstring
356             indent = " " * 4 * self.current_line.depth
357
358             if is_multiline_string(leaf):
359                 docstring = fix_docstring(docstring, indent)
360             else:
361                 docstring = docstring.strip()
362
363             if docstring:
364                 # Add some padding if the docstring starts / ends with a quote mark.
365                 if docstring[0] == quote_char:
366                     docstring = " " + docstring
367                 if docstring[-1] == quote_char:
368                     docstring += " "
369                 if docstring[-1] == "\\":
370                     backslash_count = len(docstring) - len(docstring.rstrip("\\"))
371                     if backslash_count % 2:
372                         # Odd number of tailing backslashes, add some padding to
373                         # avoid escaping the closing string quote.
374                         docstring += " "
375             elif not docstring_started_empty:
376                 docstring = " "
377
378             # We could enforce triple quotes at this point.
379             quote = quote_char * quote_len
380
381             # It's invalid to put closing single-character quotes on a new line.
382             if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
383                 # We need to find the length of the last line of the docstring
384                 # to find if we can add the closing quotes to the line without
385                 # exceeding the maximum line length.
386                 # If docstring is one line, then we need to add the length
387                 # of the indent, prefix, and starting quotes. Ending quotes are
388                 # handled later.
389                 lines = docstring.splitlines()
390                 last_line_length = len(lines[-1]) if docstring else 0
391
392                 if len(lines) == 1:
393                     last_line_length += len(indent) + len(prefix) + quote_len
394
395                 # If adding closing quotes would cause the last line to exceed
396                 # the maximum line length then put a line break before the
397                 # closing quotes
398                 if last_line_length + quote_len > self.mode.line_length:
399                     leaf.value = prefix + quote + docstring + "\n" + indent + quote
400                 else:
401                     leaf.value = prefix + quote + docstring + quote
402             else:
403                 leaf.value = prefix + quote + docstring + quote
404
405         yield from self.visit_default(leaf)
406
407     def __post_init__(self) -> None:
408         """You are in a twisty little maze of passages."""
409         self.current_line = Line(mode=self.mode)
410
411         v = self.visit_stmt
412         Ø: Set[str] = set()
413         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
414         self.visit_if_stmt = partial(
415             v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
416         )
417         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
418         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
419         self.visit_try_stmt = partial(
420             v, keywords={"try", "except", "else", "finally"}, parens=Ø
421         )
422         if self.mode.preview:
423             self.visit_except_clause = partial(
424                 v, keywords={"except"}, parens={"except"}
425             )
426             self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
427         else:
428             self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
429             self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
430         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
431         self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
432         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
433         self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
434         self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
435         self.visit_async_funcdef = self.visit_async_stmt
436         self.visit_decorated = self.visit_decorators
437
438         # PEP 634
439         self.visit_match_stmt = self.visit_match_case
440         self.visit_case_block = self.visit_match_case
441
442
443 def transform_line(
444     line: Line, mode: Mode, features: Collection[Feature] = ()
445 ) -> Iterator[Line]:
446     """Transform a `line`, potentially splitting it into many lines.
447
448     They should fit in the allotted `line_length` but might not be able to.
449
450     `features` are syntactical features that may be used in the output.
451     """
452     if line.is_comment:
453         yield line
454         return
455
456     line_str = line_to_string(line)
457
458     ll = mode.line_length
459     sn = mode.string_normalization
460     string_merge = StringMerger(ll, sn)
461     string_paren_strip = StringParenStripper(ll, sn)
462     string_split = StringSplitter(ll, sn)
463     string_paren_wrap = StringParenWrapper(ll, sn)
464
465     transformers: List[Transformer]
466     if (
467         not line.contains_uncollapsable_type_comments()
468         and not line.should_split_rhs
469         and not line.magic_trailing_comma
470         and (
471             is_line_short_enough(line, line_length=mode.line_length, line_str=line_str)
472             or line.contains_unsplittable_type_ignore()
473         )
474         and not (line.inside_brackets and line.contains_standalone_comments())
475     ):
476         # Only apply basic string preprocessing, since lines shouldn't be split here.
477         if Preview.string_processing in mode:
478             transformers = [string_merge, string_paren_strip]
479         else:
480             transformers = []
481     elif line.is_def:
482         transformers = [left_hand_split]
483     else:
484
485         def _rhs(
486             self: object, line: Line, features: Collection[Feature]
487         ) -> Iterator[Line]:
488             """Wraps calls to `right_hand_split`.
489
490             The calls increasingly `omit` right-hand trailers (bracket pairs with
491             content), meaning the trailers get glued together to split on another
492             bracket pair instead.
493             """
494             for omit in generate_trailers_to_omit(line, mode.line_length):
495                 lines = list(
496                     right_hand_split(line, mode.line_length, features, omit=omit)
497                 )
498                 # Note: this check is only able to figure out if the first line of the
499                 # *current* transformation fits in the line length.  This is true only
500                 # for simple cases.  All others require running more transforms via
501                 # `transform_line()`.  This check doesn't know if those would succeed.
502                 if is_line_short_enough(lines[0], line_length=mode.line_length):
503                     yield from lines
504                     return
505
506             # All splits failed, best effort split with no omits.
507             # This mostly happens to multiline strings that are by definition
508             # reported as not fitting a single line, as well as lines that contain
509             # trailing commas (those have to be exploded).
510             yield from right_hand_split(
511                 line, line_length=mode.line_length, features=features
512             )
513
514         # HACK: nested functions (like _rhs) compiled by mypyc don't retain their
515         # __name__ attribute which is needed in `run_transformer` further down.
516         # Unfortunately a nested class breaks mypyc too. So a class must be created
517         # via type ... https://github.com/mypyc/mypyc/issues/884
518         rhs = type("rhs", (), {"__call__": _rhs})()
519
520         if Preview.string_processing in mode:
521             if line.inside_brackets:
522                 transformers = [
523                     string_merge,
524                     string_paren_strip,
525                     string_split,
526                     delimiter_split,
527                     standalone_comment_split,
528                     string_paren_wrap,
529                     rhs,
530                 ]
531             else:
532                 transformers = [
533                     string_merge,
534                     string_paren_strip,
535                     string_split,
536                     string_paren_wrap,
537                     rhs,
538                 ]
539         else:
540             if line.inside_brackets:
541                 transformers = [delimiter_split, standalone_comment_split, rhs]
542             else:
543                 transformers = [rhs]
544     # It's always safe to attempt hugging of power operations and pretty much every line
545     # could match.
546     transformers.append(hug_power_op)
547
548     for transform in transformers:
549         # We are accumulating lines in `result` because we might want to abort
550         # mission and return the original line in the end, or attempt a different
551         # split altogether.
552         try:
553             result = run_transformer(line, transform, mode, features, line_str=line_str)
554         except CannotTransform:
555             continue
556         else:
557             yield from result
558             break
559
560     else:
561         yield line
562
563
564 def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
565     """Split line into many lines, starting with the first matching bracket pair.
566
567     Note: this usually looks weird, only use this for function definitions.
568     Prefer RHS otherwise.  This is why this function is not symmetrical with
569     :func:`right_hand_split` which also handles optional parentheses.
570     """
571     tail_leaves: List[Leaf] = []
572     body_leaves: List[Leaf] = []
573     head_leaves: List[Leaf] = []
574     current_leaves = head_leaves
575     matching_bracket: Optional[Leaf] = None
576     for leaf in line.leaves:
577         if (
578             current_leaves is body_leaves
579             and leaf.type in CLOSING_BRACKETS
580             and leaf.opening_bracket is matching_bracket
581             and isinstance(matching_bracket, Leaf)
582         ):
583             ensure_visible(leaf)
584             ensure_visible(matching_bracket)
585             current_leaves = tail_leaves if body_leaves else head_leaves
586         current_leaves.append(leaf)
587         if current_leaves is head_leaves:
588             if leaf.type in OPENING_BRACKETS:
589                 matching_bracket = leaf
590                 current_leaves = body_leaves
591     if not matching_bracket:
592         raise CannotSplit("No brackets found")
593
594     head = bracket_split_build_line(head_leaves, line, matching_bracket)
595     body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
596     tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
597     bracket_split_succeeded_or_raise(head, body, tail)
598     for result in (head, body, tail):
599         if result:
600             yield result
601
602
603 def right_hand_split(
604     line: Line,
605     line_length: int,
606     features: Collection[Feature] = (),
607     omit: Collection[LeafID] = (),
608 ) -> Iterator[Line]:
609     """Split line into many lines, starting with the last matching bracket pair.
610
611     If the split was by optional parentheses, attempt splitting without them, too.
612     `omit` is a collection of closing bracket IDs that shouldn't be considered for
613     this split.
614
615     Note: running this function modifies `bracket_depth` on the leaves of `line`.
616     """
617     tail_leaves: List[Leaf] = []
618     body_leaves: List[Leaf] = []
619     head_leaves: List[Leaf] = []
620     current_leaves = tail_leaves
621     opening_bracket: Optional[Leaf] = None
622     closing_bracket: Optional[Leaf] = None
623     for leaf in reversed(line.leaves):
624         if current_leaves is body_leaves:
625             if leaf is opening_bracket:
626                 current_leaves = head_leaves if body_leaves else tail_leaves
627         current_leaves.append(leaf)
628         if current_leaves is tail_leaves:
629             if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
630                 opening_bracket = leaf.opening_bracket
631                 closing_bracket = leaf
632                 current_leaves = body_leaves
633     if not (opening_bracket and closing_bracket and head_leaves):
634         # If there is no opening or closing_bracket that means the split failed and
635         # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means
636         # the matching `opening_bracket` wasn't available on `line` anymore.
637         raise CannotSplit("No brackets found")
638
639     tail_leaves.reverse()
640     body_leaves.reverse()
641     head_leaves.reverse()
642     head = bracket_split_build_line(head_leaves, line, opening_bracket)
643     body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
644     tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
645     bracket_split_succeeded_or_raise(head, body, tail)
646     if (
647         Feature.FORCE_OPTIONAL_PARENTHESES not in features
648         # the opening bracket is an optional paren
649         and opening_bracket.type == token.LPAR
650         and not opening_bracket.value
651         # the closing bracket is an optional paren
652         and closing_bracket.type == token.RPAR
653         and not closing_bracket.value
654         # it's not an import (optional parens are the only thing we can split on
655         # in this case; attempting a split without them is a waste of time)
656         and not line.is_import
657         # there are no standalone comments in the body
658         and not body.contains_standalone_comments(0)
659         # and we can actually remove the parens
660         and can_omit_invisible_parens(body, line_length)
661     ):
662         omit = {id(closing_bracket), *omit}
663         try:
664             yield from right_hand_split(line, line_length, features=features, omit=omit)
665             return
666
667         except CannotSplit as e:
668             if not (
669                 can_be_split(body)
670                 or is_line_short_enough(body, line_length=line_length)
671             ):
672                 raise CannotSplit(
673                     "Splitting failed, body is still too long and can't be split."
674                 ) from e
675
676             elif head.contains_multiline_strings() or tail.contains_multiline_strings():
677                 raise CannotSplit(
678                     "The current optional pair of parentheses is bound to fail to"
679                     " satisfy the splitting algorithm because the head or the tail"
680                     " contains multiline strings which by definition never fit one"
681                     " line."
682                 ) from e
683
684     ensure_visible(opening_bracket)
685     ensure_visible(closing_bracket)
686     for result in (head, body, tail):
687         if result:
688             yield result
689
690
691 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
692     """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
693
694     Do nothing otherwise.
695
696     A left- or right-hand split is based on a pair of brackets. Content before
697     (and including) the opening bracket is left on one line, content inside the
698     brackets is put on a separate line, and finally content starting with and
699     following the closing bracket is put on a separate line.
700
701     Those are called `head`, `body`, and `tail`, respectively. If the split
702     produced the same line (all content in `head`) or ended up with an empty `body`
703     and the `tail` is just the closing bracket, then it's considered failed.
704     """
705     tail_len = len(str(tail).strip())
706     if not body:
707         if tail_len == 0:
708             raise CannotSplit("Splitting brackets produced the same line")
709
710         elif tail_len < 3:
711             raise CannotSplit(
712                 f"Splitting brackets on an empty body to save {tail_len} characters is"
713                 " not worth it"
714             )
715
716
717 def bracket_split_build_line(
718     leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
719 ) -> Line:
720     """Return a new line with given `leaves` and respective comments from `original`.
721
722     If `is_body` is True, the result line is one-indented inside brackets and as such
723     has its first leaf's prefix normalized and a trailing comma added when expected.
724     """
725     result = Line(mode=original.mode, depth=original.depth)
726     if is_body:
727         result.inside_brackets = True
728         result.depth += 1
729         if leaves:
730             # Since body is a new indent level, remove spurious leading whitespace.
731             normalize_prefix(leaves[0], inside_brackets=True)
732             # Ensure a trailing comma for imports and standalone function arguments, but
733             # be careful not to add one after any comments or within type annotations.
734             no_commas = (
735                 original.is_def
736                 and opening_bracket.value == "("
737                 and not any(leaf.type == token.COMMA for leaf in leaves)
738                 # In particular, don't add one within a parenthesized return annotation.
739                 # Unfortunately the indicator we're in a return annotation (RARROW) may
740                 # be defined directly in the parent node, the parent of the parent ...
741                 # and so on depending on how complex the return annotation is.
742                 # This isn't perfect and there's some false negatives but they are in
743                 # contexts were a comma is actually fine.
744                 and not any(
745                     node.prev_sibling.type == RARROW
746                     for node in (
747                         leaves[0].parent,
748                         getattr(leaves[0].parent, "parent", None),
749                     )
750                     if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf)
751                 )
752             )
753
754             if original.is_import or no_commas:
755                 for i in range(len(leaves) - 1, -1, -1):
756                     if leaves[i].type == STANDALONE_COMMENT:
757                         continue
758
759                     if leaves[i].type != token.COMMA:
760                         new_comma = Leaf(token.COMMA, ",")
761                         leaves.insert(i + 1, new_comma)
762                     break
763
764     # Populate the line
765     for leaf in leaves:
766         result.append(leaf, preformatted=True)
767         for comment_after in original.comments_after(leaf):
768             result.append(comment_after, preformatted=True)
769     if is_body and should_split_line(result, opening_bracket):
770         result.should_split_rhs = True
771     return result
772
773
774 def dont_increase_indentation(split_func: Transformer) -> Transformer:
775     """Normalize prefix of the first leaf in every line returned by `split_func`.
776
777     This is a decorator over relevant split functions.
778     """
779
780     @wraps(split_func)
781     def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
782         for split_line in split_func(line, features):
783             normalize_prefix(split_line.leaves[0], inside_brackets=True)
784             yield split_line
785
786     return split_wrapper
787
788
789 @dont_increase_indentation
790 def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
791     """Split according to delimiters of the highest priority.
792
793     If the appropriate Features are given, the split will add trailing commas
794     also in function signatures and calls that contain `*` and `**`.
795     """
796     try:
797         last_leaf = line.leaves[-1]
798     except IndexError:
799         raise CannotSplit("Line empty") from None
800
801     bt = line.bracket_tracker
802     try:
803         delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
804     except ValueError:
805         raise CannotSplit("No delimiters found") from None
806
807     if delimiter_priority == DOT_PRIORITY:
808         if bt.delimiter_count_with_priority(delimiter_priority) == 1:
809             raise CannotSplit("Splitting a single attribute from its owner looks wrong")
810
811     current_line = Line(
812         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
813     )
814     lowest_depth = sys.maxsize
815     trailing_comma_safe = True
816
817     def append_to_line(leaf: Leaf) -> Iterator[Line]:
818         """Append `leaf` to current line or to new line if appending impossible."""
819         nonlocal current_line
820         try:
821             current_line.append_safe(leaf, preformatted=True)
822         except ValueError:
823             yield current_line
824
825             current_line = Line(
826                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
827             )
828             current_line.append(leaf)
829
830     for leaf in line.leaves:
831         yield from append_to_line(leaf)
832
833         for comment_after in line.comments_after(leaf):
834             yield from append_to_line(comment_after)
835
836         lowest_depth = min(lowest_depth, leaf.bracket_depth)
837         if leaf.bracket_depth == lowest_depth:
838             if is_vararg(leaf, within={syms.typedargslist}):
839                 trailing_comma_safe = (
840                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
841                 )
842             elif is_vararg(leaf, within={syms.arglist, syms.argument}):
843                 trailing_comma_safe = (
844                     trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
845                 )
846
847         leaf_priority = bt.delimiters.get(id(leaf))
848         if leaf_priority == delimiter_priority:
849             yield current_line
850
851             current_line = Line(
852                 mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
853             )
854     if current_line:
855         if (
856             trailing_comma_safe
857             and delimiter_priority == COMMA_PRIORITY
858             and current_line.leaves[-1].type != token.COMMA
859             and current_line.leaves[-1].type != STANDALONE_COMMENT
860         ):
861             new_comma = Leaf(token.COMMA, ",")
862             current_line.append(new_comma)
863         yield current_line
864
865
866 @dont_increase_indentation
867 def standalone_comment_split(
868     line: Line, features: Collection[Feature] = ()
869 ) -> Iterator[Line]:
870     """Split standalone comments from the rest of the line."""
871     if not line.contains_standalone_comments(0):
872         raise CannotSplit("Line does not have any standalone comments")
873
874     current_line = Line(
875         mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets
876     )
877
878     def append_to_line(leaf: Leaf) -> Iterator[Line]:
879         """Append `leaf` to current line or to new line if appending impossible."""
880         nonlocal current_line
881         try:
882             current_line.append_safe(leaf, preformatted=True)
883         except ValueError:
884             yield current_line
885
886             current_line = Line(
887                 line.mode, depth=line.depth, inside_brackets=line.inside_brackets
888             )
889             current_line.append(leaf)
890
891     for leaf in line.leaves:
892         yield from append_to_line(leaf)
893
894         for comment_after in line.comments_after(leaf):
895             yield from append_to_line(comment_after)
896
897     if current_line:
898         yield current_line
899
900
901 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
902     """Leave existing extra newlines if not `inside_brackets`. Remove everything
903     else.
904
905     Note: don't use backslashes for formatting or you'll lose your voting rights.
906     """
907     if not inside_brackets:
908         spl = leaf.prefix.split("#")
909         if "\\" not in spl[0]:
910             nl_count = spl[-1].count("\n")
911             if len(spl) > 1:
912                 nl_count -= 1
913             leaf.prefix = "\n" * nl_count
914             return
915
916     leaf.prefix = ""
917
918
919 def normalize_invisible_parens(
920     node: Node, parens_after: Set[str], *, preview: bool
921 ) -> None:
922     """Make existing optional parentheses invisible or create new ones.
923
924     `parens_after` is a set of string leaf values immediately after which parens
925     should be put.
926
927     Standardizes on visible parentheses for single-element tuples, and keeps
928     existing visible parentheses for other tuples and generator expressions.
929     """
930     for pc in list_comments(node.prefix, is_endmarker=False, preview=preview):
931         if pc.value in FMT_OFF:
932             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
933             return
934     check_lpar = False
935     for index, child in enumerate(list(node.children)):
936         # Fixes a bug where invisible parens are not properly stripped from
937         # assignment statements that contain type annotations.
938         if isinstance(child, Node) and child.type == syms.annassign:
939             normalize_invisible_parens(
940                 child, parens_after=parens_after, preview=preview
941             )
942
943         # Add parentheses around long tuple unpacking in assignments.
944         if (
945             index == 0
946             and isinstance(child, Node)
947             and child.type == syms.testlist_star_expr
948         ):
949             check_lpar = True
950
951         if check_lpar:
952             if (
953                 preview
954                 and child.type == syms.atom
955                 and node.type == syms.for_stmt
956                 and isinstance(child.prev_sibling, Leaf)
957                 and child.prev_sibling.type == token.NAME
958                 and child.prev_sibling.value == "for"
959             ):
960                 if maybe_make_parens_invisible_in_atom(
961                     child,
962                     parent=node,
963                     remove_brackets_around_comma=True,
964                 ):
965                     wrap_in_parentheses(node, child, visible=False)
966             elif preview and isinstance(child, Node) and node.type == syms.with_stmt:
967                 remove_with_parens(child, node)
968             elif child.type == syms.atom:
969                 if maybe_make_parens_invisible_in_atom(
970                     child,
971                     parent=node,
972                 ):
973                     wrap_in_parentheses(node, child, visible=False)
974             elif is_one_tuple(child):
975                 wrap_in_parentheses(node, child, visible=True)
976             elif node.type == syms.import_from:
977                 # "import from" nodes store parentheses directly as part of
978                 # the statement
979                 if is_lpar_token(child):
980                     assert is_rpar_token(node.children[-1])
981                     # make parentheses invisible
982                     child.value = ""
983                     node.children[-1].value = ""
984                 elif child.type != token.STAR:
985                     # insert invisible parentheses
986                     node.insert_child(index, Leaf(token.LPAR, ""))
987                     node.append_child(Leaf(token.RPAR, ""))
988                 break
989             elif (
990                 index == 1
991                 and child.type == token.STAR
992                 and node.type == syms.except_clause
993             ):
994                 # In except* (PEP 654), the star is actually part of
995                 # of the keyword. So we need to skip the insertion of
996                 # invisible parentheses to work more precisely.
997                 continue
998
999             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
1000                 wrap_in_parentheses(node, child, visible=False)
1001
1002         comma_check = child.type == token.COMMA if preview else False
1003
1004         check_lpar = isinstance(child, Leaf) and (
1005             child.value in parens_after or comma_check
1006         )
1007
1008
1009 def remove_await_parens(node: Node) -> None:
1010     if node.children[0].type == token.AWAIT and len(node.children) > 1:
1011         if (
1012             node.children[1].type == syms.atom
1013             and node.children[1].children[0].type == token.LPAR
1014         ):
1015             if maybe_make_parens_invisible_in_atom(
1016                 node.children[1],
1017                 parent=node,
1018                 remove_brackets_around_comma=True,
1019             ):
1020                 wrap_in_parentheses(node, node.children[1], visible=False)
1021
1022             # Since await is an expression we shouldn't remove
1023             # brackets in cases where this would change
1024             # the AST due to operator precedence.
1025             # Therefore we only aim to remove brackets around
1026             # power nodes that aren't also await expressions themselves.
1027             # https://peps.python.org/pep-0492/#updated-operator-precedence-table
1028             # N.B. We've still removed any redundant nested brackets though :)
1029             opening_bracket = cast(Leaf, node.children[1].children[0])
1030             closing_bracket = cast(Leaf, node.children[1].children[-1])
1031             bracket_contents = cast(Node, node.children[1].children[1])
1032             if bracket_contents.type != syms.power:
1033                 ensure_visible(opening_bracket)
1034                 ensure_visible(closing_bracket)
1035             elif (
1036                 bracket_contents.type == syms.power
1037                 and bracket_contents.children[0].type == token.AWAIT
1038             ):
1039                 ensure_visible(opening_bracket)
1040                 ensure_visible(closing_bracket)
1041                 # If we are in a nested await then recurse down.
1042                 remove_await_parens(bracket_contents)
1043
1044
1045 def remove_with_parens(node: Node, parent: Node) -> None:
1046     """Recursively hide optional parens in `with` statements."""
1047     # Removing all unnecessary parentheses in with statements in one pass is a tad
1048     # complex as different variations of bracketed statements result in pretty
1049     # different parse trees:
1050     #
1051     # with (open("file")) as f:                       # this is an asexpr_test
1052     #     ...
1053     #
1054     # with (open("file") as f):                       # this is an atom containing an
1055     #     ...                                         # asexpr_test
1056     #
1057     # with (open("file")) as f, (open("file")) as f:  # this is asexpr_test, COMMA,
1058     #     ...                                         # asexpr_test
1059     #
1060     # with (open("file") as f, open("file") as f):    # an atom containing a
1061     #     ...                                         # testlist_gexp which then
1062     #                                                 # contains multiple asexpr_test(s)
1063     if node.type == syms.atom:
1064         if maybe_make_parens_invisible_in_atom(
1065             node,
1066             parent=parent,
1067             remove_brackets_around_comma=True,
1068         ):
1069             wrap_in_parentheses(parent, node, visible=False)
1070         if isinstance(node.children[1], Node):
1071             remove_with_parens(node.children[1], node)
1072     elif node.type == syms.testlist_gexp:
1073         for child in node.children:
1074             if isinstance(child, Node):
1075                 remove_with_parens(child, node)
1076     elif node.type == syms.asexpr_test and not any(
1077         leaf.type == token.COLONEQUAL for leaf in node.leaves()
1078     ):
1079         if maybe_make_parens_invisible_in_atom(
1080             node.children[0],
1081             parent=node,
1082             remove_brackets_around_comma=True,
1083         ):
1084             wrap_in_parentheses(node, node.children[0], visible=False)
1085
1086
1087 def maybe_make_parens_invisible_in_atom(
1088     node: LN,
1089     parent: LN,
1090     remove_brackets_around_comma: bool = False,
1091 ) -> bool:
1092     """If it's safe, make the parens in the atom `node` invisible, recursively.
1093     Additionally, remove repeated, adjacent invisible parens from the atom `node`
1094     as they are redundant.
1095
1096     Returns whether the node should itself be wrapped in invisible parentheses.
1097     """
1098     if (
1099         node.type != syms.atom
1100         or is_empty_tuple(node)
1101         or is_one_tuple(node)
1102         or (is_yield(node) and parent.type != syms.expr_stmt)
1103         or (
1104             # This condition tries to prevent removing non-optional brackets
1105             # around a tuple, however, can be a bit overzealous so we provide
1106             # and option to skip this check for `for` and `with` statements.
1107             not remove_brackets_around_comma
1108             and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
1109         )
1110     ):
1111         return False
1112
1113     if is_walrus_assignment(node):
1114         if parent.type in [
1115             syms.annassign,
1116             syms.expr_stmt,
1117             syms.assert_stmt,
1118             syms.return_stmt,
1119             # these ones aren't useful to end users, but they do please fuzzers
1120             syms.for_stmt,
1121             syms.del_stmt,
1122         ]:
1123             return False
1124
1125     first = node.children[0]
1126     last = node.children[-1]
1127     if is_lpar_token(first) and is_rpar_token(last):
1128         middle = node.children[1]
1129         # make parentheses invisible
1130         first.value = ""
1131         last.value = ""
1132         maybe_make_parens_invisible_in_atom(
1133             middle,
1134             parent=parent,
1135             remove_brackets_around_comma=remove_brackets_around_comma,
1136         )
1137
1138         if is_atom_with_invisible_parens(middle):
1139             # Strip the invisible parens from `middle` by replacing
1140             # it with the child in-between the invisible parens
1141             middle.replace(middle.children[1])
1142
1143         return False
1144
1145     return True
1146
1147
1148 def should_split_line(line: Line, opening_bracket: Leaf) -> bool:
1149     """Should `line` be immediately split with `delimiter_split()` after RHS?"""
1150
1151     if not (opening_bracket.parent and opening_bracket.value in "[{("):
1152         return False
1153
1154     # We're essentially checking if the body is delimited by commas and there's more
1155     # than one of them (we're excluding the trailing comma and if the delimiter priority
1156     # is still commas, that means there's more).
1157     exclude = set()
1158     trailing_comma = False
1159     try:
1160         last_leaf = line.leaves[-1]
1161         if last_leaf.type == token.COMMA:
1162             trailing_comma = True
1163             exclude.add(id(last_leaf))
1164         max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
1165     except (IndexError, ValueError):
1166         return False
1167
1168     return max_priority == COMMA_PRIORITY and (
1169         (line.mode.magic_trailing_comma and trailing_comma)
1170         # always explode imports
1171         or opening_bracket.parent.type in {syms.atom, syms.import_from}
1172     )
1173
1174
1175 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
1176     """Generate sets of closing bracket IDs that should be omitted in a RHS.
1177
1178     Brackets can be omitted if the entire trailer up to and including
1179     a preceding closing bracket fits in one line.
1180
1181     Yielded sets are cumulative (contain results of previous yields, too).  First
1182     set is empty, unless the line should explode, in which case bracket pairs until
1183     the one that needs to explode are omitted.
1184     """
1185
1186     omit: Set[LeafID] = set()
1187     if not line.magic_trailing_comma:
1188         yield omit
1189
1190     length = 4 * line.depth
1191     opening_bracket: Optional[Leaf] = None
1192     closing_bracket: Optional[Leaf] = None
1193     inner_brackets: Set[LeafID] = set()
1194     for index, leaf, leaf_length in line.enumerate_with_length(reversed=True):
1195         length += leaf_length
1196         if length > line_length:
1197             break
1198
1199         has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
1200         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
1201             break
1202
1203         if opening_bracket:
1204             if leaf is opening_bracket:
1205                 opening_bracket = None
1206             elif leaf.type in CLOSING_BRACKETS:
1207                 prev = line.leaves[index - 1] if index > 0 else None
1208                 if (
1209                     prev
1210                     and prev.type == token.COMMA
1211                     and leaf.opening_bracket is not None
1212                     and not is_one_sequence_between(
1213                         leaf.opening_bracket, leaf, line.leaves
1214                     )
1215                 ):
1216                     # Never omit bracket pairs with trailing commas.
1217                     # We need to explode on those.
1218                     break
1219
1220                 inner_brackets.add(id(leaf))
1221         elif leaf.type in CLOSING_BRACKETS:
1222             prev = line.leaves[index - 1] if index > 0 else None
1223             if prev and prev.type in OPENING_BRACKETS:
1224                 # Empty brackets would fail a split so treat them as "inner"
1225                 # brackets (e.g. only add them to the `omit` set if another
1226                 # pair of brackets was good enough.
1227                 inner_brackets.add(id(leaf))
1228                 continue
1229
1230             if closing_bracket:
1231                 omit.add(id(closing_bracket))
1232                 omit.update(inner_brackets)
1233                 inner_brackets.clear()
1234                 yield omit
1235
1236             if (
1237                 prev
1238                 and prev.type == token.COMMA
1239                 and leaf.opening_bracket is not None
1240                 and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)
1241             ):
1242                 # Never omit bracket pairs with trailing commas.
1243                 # We need to explode on those.
1244                 break
1245
1246             if leaf.value:
1247                 opening_bracket = leaf.opening_bracket
1248                 closing_bracket = leaf
1249
1250
1251 def run_transformer(
1252     line: Line,
1253     transform: Transformer,
1254     mode: Mode,
1255     features: Collection[Feature],
1256     *,
1257     line_str: str = "",
1258 ) -> List[Line]:
1259     if not line_str:
1260         line_str = line_to_string(line)
1261     result: List[Line] = []
1262     for transformed_line in transform(line, features):
1263         if str(transformed_line).strip("\n") == line_str:
1264             raise CannotTransform("Line transformer returned an unchanged result")
1265
1266         result.extend(transform_line(transformed_line, mode=mode, features=features))
1267
1268     if (
1269         transform.__class__.__name__ != "rhs"
1270         or not line.bracket_tracker.invisible
1271         or any(bracket.value for bracket in line.bracket_tracker.invisible)
1272         or line.contains_multiline_strings()
1273         or result[0].contains_uncollapsable_type_comments()
1274         or result[0].contains_unsplittable_type_ignore()
1275         or is_line_short_enough(result[0], line_length=mode.line_length)
1276         # If any leaves have no parents (which _can_ occur since
1277         # `transform(line)` potentially destroys the line's underlying node
1278         # structure), then we can't proceed. Doing so would cause the below
1279         # call to `append_leaves()` to fail.
1280         or any(leaf.parent is None for leaf in line.leaves)
1281     ):
1282         return result
1283
1284     line_copy = line.clone()
1285     append_leaves(line_copy, line, line.leaves)
1286     features_fop = set(features) | {Feature.FORCE_OPTIONAL_PARENTHESES}
1287     second_opinion = run_transformer(
1288         line_copy, transform, mode, features_fop, line_str=line_str
1289     )
1290     if all(
1291         is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion
1292     ):
1293         result = second_opinion
1294     return result