All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
3 from asyncio.base_events import BaseEventLoop
4 from concurrent.futures import Executor, ProcessPoolExecutor
6 from functools import partial, wraps
9 from multiprocessing import Manager
11 from pathlib import Path
35 from appdirs import user_cache_dir
36 from attr import dataclass, Factory
40 from blib2to3.pytree import Node, Leaf, type_repr
41 from blib2to3 import pygram, pytree
42 from blib2to3.pgen2 import driver, token
43 from blib2to3.pgen2.parse import ParseError
46 __version__ = "18.4a6"
47 DEFAULT_LINE_LENGTH = 88
50 syms = pygram.python_symbols
58 LN = Union[Leaf, Node]
59 SplitFunc = Callable[["Line", bool], Iterator["Line"]]
62 CacheInfo = Tuple[Timestamp, FileSize]
63 Cache = Dict[Path, CacheInfo]
64 out = partial(click.secho, bold=True, err=True)
65 err = partial(click.secho, fg="red", err=True)
68 class NothingChanged(UserWarning):
69 """Raised by :func:`format_file` when reformatted code is the same as source."""
72 class CannotSplit(Exception):
73 """A readable split that fits the allotted line length is impossible.
75 Raised by :func:`left_hand_split`, :func:`right_hand_split`, and
76 :func:`delimiter_split`.
80 class FormatError(Exception):
81 """Base exception for `# fmt: on` and `# fmt: off` handling.
83 It holds the number of bytes of the prefix consumed before the format
84 control comment appeared.
87 def __init__(self, consumed: int) -> None:
88 super().__init__(consumed)
89 self.consumed = consumed
91 def trim_prefix(self, leaf: Leaf) -> None:
92 leaf.prefix = leaf.prefix[self.consumed :]
94 def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
95 """Returns a new Leaf from the consumed part of the prefix."""
96 unformatted_prefix = leaf.prefix[: self.consumed]
97 return Leaf(token.NEWLINE, unformatted_prefix)
100 class FormatOn(FormatError):
101 """Found a comment like `# fmt: on` in the file."""
104 class FormatOff(FormatError):
105 """Found a comment like `# fmt: off` in the file."""
108 class WriteBack(Enum):
125 default=DEFAULT_LINE_LENGTH,
126 help="How many character per line to allow.",
133 "Don't write the files back, just return the status. Return code 0 "
134 "means nothing would change. Return code 1 means some files would be "
135 "reformatted. Return code 123 means there was an internal error."
141 help="Don't write the files back, just output a diff for each file on stdout.",
146 help="If --fast given, skip temporary sanity checks. [default: --safe]",
153 "Don't emit non-error messages to stderr. Errors are still emitted, "
154 "silence those with 2>/dev/null."
157 @click.version_option(version=__version__)
162 exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
175 """The uncompromising code formatter."""
176 sources: List[Path] = []
180 sources.extend(gen_python_files_in_dir(p))
182 # if a file was explicitly given, we don't care about its extension
185 sources.append(Path("-"))
187 err(f"invalid path: {s}")
189 if check and not diff:
190 write_back = WriteBack.NO
192 write_back = WriteBack.DIFF
194 write_back = WriteBack.YES
195 report = Report(check=check, quiet=quiet)
196 if len(sources) == 0:
197 out("No paths given. Nothing to do 😴")
201 elif len(sources) == 1:
202 reformat_one(sources[0], line_length, fast, write_back, report)
204 loop = asyncio.get_event_loop()
205 executor = ProcessPoolExecutor(max_workers=os.cpu_count())
207 loop.run_until_complete(
209 sources, line_length, fast, write_back, report, loop, executor
215 out("All done! ✨ 🍰 ✨")
216 click.echo(str(report))
217 ctx.exit(report.return_code)
221 src: Path, line_length: int, fast: bool, write_back: WriteBack, report: "Report"
223 """Reformat a single file under `src` without spawning child processes.
225 If `quiet` is True, non-error messages are not output. `line_length`,
226 `write_back`, and `fast` options are passed to :func:`format_file_in_place`.
230 if not src.is_file() and str(src) == "-":
231 if format_stdin_to_stdout(
232 line_length=line_length, fast=fast, write_back=write_back
234 changed = Changed.YES
237 if write_back != WriteBack.DIFF:
238 cache = read_cache(line_length)
240 if src in cache and cache[src] == get_cache_info(src):
241 changed = Changed.CACHED
243 changed is not Changed.CACHED
244 and format_file_in_place(
245 src, line_length=line_length, fast=fast, write_back=write_back
248 changed = Changed.YES
249 if write_back == WriteBack.YES and changed is not Changed.NO:
250 write_cache(cache, [src], line_length)
251 report.done(src, changed)
252 except Exception as exc:
253 report.failed(src, str(exc))
256 async def schedule_formatting(
260 write_back: WriteBack,
265 """Run formatting of `sources` in parallel using the provided `executor`.
267 (Use ProcessPoolExecutors for actual parallelism.)
269 `line_length`, `write_back`, and `fast` options are passed to
270 :func:`format_file_in_place`.
273 if write_back != WriteBack.DIFF:
274 cache = read_cache(line_length)
275 sources, cached = filter_cached(cache, sources)
277 report.done(src, Changed.CACHED)
282 if write_back == WriteBack.DIFF:
283 # For diff output, we need locks to ensure we don't interleave output
284 # from different processes.
286 lock = manager.Lock()
288 src: loop.run_in_executor(
289 executor, format_file_in_place, src, line_length, fast, write_back, lock
293 _task_values = list(tasks.values())
295 loop.add_signal_handler(signal.SIGINT, cancel, _task_values)
296 loop.add_signal_handler(signal.SIGTERM, cancel, _task_values)
297 except NotImplementedError:
298 # There are no good alternatives for these on Windows
300 await asyncio.wait(_task_values)
301 for src, task in tasks.items():
303 report.failed(src, "timed out, cancelling")
305 cancelled.append(task)
306 elif task.cancelled():
307 cancelled.append(task)
308 elif task.exception():
309 report.failed(src, str(task.exception()))
311 formatted.append(src)
312 report.done(src, Changed.YES if task.result() else Changed.NO)
315 await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
316 if write_back == WriteBack.YES and formatted:
317 write_cache(cache, formatted, line_length)
320 def format_file_in_place(
324 write_back: WriteBack = WriteBack.NO,
325 lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy
327 """Format file under `src` path. Return True if changed.
329 If `write_back` is True, write reformatted code back to stdout.
330 `line_length` and `fast` options are passed to :func:`format_file_contents`.
332 is_pyi = src.suffix == ".pyi"
334 with tokenize.open(src) as src_buffer:
335 src_contents = src_buffer.read()
337 dst_contents = format_file_contents(
338 src_contents, line_length=line_length, fast=fast, is_pyi=is_pyi
340 except NothingChanged:
343 if write_back == write_back.YES:
344 with open(src, "w", encoding=src_buffer.encoding) as f:
345 f.write(dst_contents)
346 elif write_back == write_back.DIFF:
347 src_name = f"{src} (original)"
348 dst_name = f"{src} (formatted)"
349 diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
353 sys.stdout.write(diff_contents)
360 def format_stdin_to_stdout(
361 line_length: int, fast: bool, write_back: WriteBack = WriteBack.NO
363 """Format file on stdin. Return True if changed.
365 If `write_back` is True, write reformatted code back to stdout.
366 `line_length` and `fast` arguments are passed to :func:`format_file_contents`.
368 src = sys.stdin.read()
371 dst = format_file_contents(src, line_length=line_length, fast=fast)
374 except NothingChanged:
378 if write_back == WriteBack.YES:
379 sys.stdout.write(dst)
380 elif write_back == WriteBack.DIFF:
381 src_name = "<stdin> (original)"
382 dst_name = "<stdin> (formatted)"
383 sys.stdout.write(diff(src, dst, src_name, dst_name))
386 def format_file_contents(
387 src_contents: str, *, line_length: int, fast: bool, is_pyi: bool = False
389 """Reformat contents a file and return new contents.
391 If `fast` is False, additionally confirm that the reformatted code is
392 valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
393 `line_length` is passed to :func:`format_str`.
395 if src_contents.strip() == "":
398 dst_contents = format_str(src_contents, line_length=line_length, is_pyi=is_pyi)
399 if src_contents == dst_contents:
403 assert_equivalent(src_contents, dst_contents)
405 src_contents, dst_contents, line_length=line_length, is_pyi=is_pyi
411 src_contents: str, line_length: int, *, is_pyi: bool = False
413 """Reformat a string and return new contents.
415 `line_length` determines how many characters per line are allowed.
417 src_node = lib2to3_parse(src_contents)
419 future_imports = get_future_imports(src_node)
420 elt = EmptyLineTracker(is_pyi=is_pyi)
421 py36 = is_python36(src_node)
422 lines = LineGenerator(
423 remove_u_prefix=py36 or "unicode_literals" in future_imports, is_pyi=is_pyi
427 for current_line in lines.visit(src_node):
428 for _ in range(after):
429 dst_contents += str(empty_line)
430 before, after = elt.maybe_empty_lines(current_line)
431 for _ in range(before):
432 dst_contents += str(empty_line)
433 for line in split_line(current_line, line_length=line_length, py36=py36):
434 dst_contents += str(line)
439 pygram.python_grammar_no_print_statement_no_exec_statement,
440 pygram.python_grammar_no_print_statement,
441 pygram.python_grammar,
445 def lib2to3_parse(src_txt: str) -> Node:
446 """Given a string with source, return the lib2to3 Node."""
447 grammar = pygram.python_grammar_no_print_statement
448 if src_txt[-1] != "\n":
449 nl = "\r\n" if "\r\n" in src_txt[:1024] else "\n"
451 for grammar in GRAMMARS:
452 drv = driver.Driver(grammar, pytree.convert)
454 result = drv.parse_string(src_txt, True)
457 except ParseError as pe:
458 lineno, column = pe.context[1]
459 lines = src_txt.splitlines()
461 faulty_line = lines[lineno - 1]
463 faulty_line = "<line number missing in source>"
464 exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
468 if isinstance(result, Leaf):
469 result = Node(syms.file_input, [result])
473 def lib2to3_unparse(node: Node) -> str:
474 """Given a lib2to3 node, return its string representation."""
482 class Visitor(Generic[T]):
483 """Basic lib2to3 visitor that yields things of type `T` on `visit()`."""
485 def visit(self, node: LN) -> Iterator[T]:
486 """Main method to visit `node` and its children.
488 It tries to find a `visit_*()` method for the given `node.type`, like
489 `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.
490 If no dedicated `visit_*()` method is found, chooses `visit_default()`
493 Then yields objects of type `T` from the selected visitor.
496 name = token.tok_name[node.type]
498 name = type_repr(node.type)
499 yield from getattr(self, f"visit_{name}", self.visit_default)(node)
501 def visit_default(self, node: LN) -> Iterator[T]:
502 """Default `visit_*()` implementation. Recurses to children of `node`."""
503 if isinstance(node, Node):
504 for child in node.children:
505 yield from self.visit(child)
509 class DebugVisitor(Visitor[T]):
512 def visit_default(self, node: LN) -> Iterator[T]:
513 indent = " " * (2 * self.tree_depth)
514 if isinstance(node, Node):
515 _type = type_repr(node.type)
516 out(f"{indent}{_type}", fg="yellow")
518 for child in node.children:
519 yield from self.visit(child)
522 out(f"{indent}/{_type}", fg="yellow", bold=False)
524 _type = token.tok_name.get(node.type, str(node.type))
525 out(f"{indent}{_type}", fg="blue", nl=False)
527 # We don't have to handle prefixes for `Node` objects since
528 # that delegates to the first child anyway.
529 out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
530 out(f" {node.value!r}", fg="blue", bold=False)
533 def show(cls, code: str) -> None:
534 """Pretty-print the lib2to3 AST of a given string of `code`.
536 Convenience method for debugging.
538 v: DebugVisitor[None] = DebugVisitor()
539 list(v.visit(lib2to3_parse(code)))
542 KEYWORDS = set(keyword.kwlist)
543 WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
544 FLOW_CONTROL = {"return", "raise", "break", "continue"}
555 STANDALONE_COMMENT = 153
556 LOGIC_OPERATORS = {"and", "or"}
581 STARS = {token.STAR, token.DOUBLESTAR}
584 syms.argument, # double star in arglist
585 syms.trailer, # single argument to call
587 syms.varargslist, # lambdas
589 UNPACKING_PARENTS = {
590 syms.atom, # single element of a list or set literal
628 COMPREHENSION_PRIORITY = 20
630 TERNARY_PRIORITY = 16
633 COMPARATOR_PRIORITY = 10
644 token.DOUBLESLASH: 3,
653 class BracketTracker:
654 """Keeps track of brackets on a line."""
657 bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
658 delimiters: Dict[LeafID, Priority] = Factory(dict)
659 previous: Optional[Leaf] = None
660 _for_loop_variable: int = 0
661 _lambda_arguments: int = 0
663 def mark(self, leaf: Leaf) -> None:
664 """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
666 All leaves receive an int `bracket_depth` field that stores how deep
667 within brackets a given leaf is. 0 means there are no enclosing brackets
668 that started on this line.
670 If a leaf is itself a closing bracket, it receives an `opening_bracket`
671 field that it forms a pair with. This is a one-directional link to
672 avoid reference cycles.
674 If a leaf is a delimiter (a token on which Black can split the line if
675 needed) and it's on depth 0, its `id()` is stored in the tracker's
678 if leaf.type == token.COMMENT:
681 self.maybe_decrement_after_for_loop_variable(leaf)
682 self.maybe_decrement_after_lambda_arguments(leaf)
683 if leaf.type in CLOSING_BRACKETS:
685 opening_bracket = self.bracket_match.pop((self.depth, leaf.type))
686 leaf.opening_bracket = opening_bracket
687 leaf.bracket_depth = self.depth
689 delim = is_split_before_delimiter(leaf, self.previous)
690 if delim and self.previous is not None:
691 self.delimiters[id(self.previous)] = delim
693 delim = is_split_after_delimiter(leaf, self.previous)
695 self.delimiters[id(leaf)] = delim
696 if leaf.type in OPENING_BRACKETS:
697 self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
700 self.maybe_increment_lambda_arguments(leaf)
701 self.maybe_increment_for_loop_variable(leaf)
703 def any_open_brackets(self) -> bool:
704 """Return True if there is an yet unmatched open bracket on the line."""
705 return bool(self.bracket_match)
707 def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
708 """Return the highest priority of a delimiter found on the line.
710 Values are consistent with what `is_split_*_delimiter()` return.
711 Raises ValueError on no delimiters.
713 return max(v for k, v in self.delimiters.items() if k not in exclude)
715 def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
716 """In a for loop, or comprehension, the variables are often unpacks.
718 To avoid splitting on the comma in this situation, increase the depth of
719 tokens between `for` and `in`.
721 if leaf.type == token.NAME and leaf.value == "for":
723 self._for_loop_variable += 1
728 def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
729 """See `maybe_increment_for_loop_variable` above for explanation."""
730 if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
732 self._for_loop_variable -= 1
737 def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool:
738 """In a lambda expression, there might be more than one argument.
740 To avoid splitting on the comma in this situation, increase the depth of
741 tokens between `lambda` and `:`.
743 if leaf.type == token.NAME and leaf.value == "lambda":
745 self._lambda_arguments += 1
750 def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
751 """See `maybe_increment_lambda_arguments` above for explanation."""
752 if self._lambda_arguments and leaf.type == token.COLON:
754 self._lambda_arguments -= 1
759 def get_open_lsqb(self) -> Optional[Leaf]:
760 """Return the most recent opening square bracket (if any)."""
761 return self.bracket_match.get((self.depth - 1, token.RSQB))
766 """Holds leaves and comments. Can be printed with `str(line)`."""
769 leaves: List[Leaf] = Factory(list)
770 comments: List[Tuple[Index, Leaf]] = Factory(list)
771 bracket_tracker: BracketTracker = Factory(BracketTracker)
772 inside_brackets: bool = False
774 def append(self, leaf: Leaf, preformatted: bool = False) -> None:
775 """Add a new `leaf` to the end of the line.
777 Unless `preformatted` is True, the `leaf` will receive a new consistent
778 whitespace prefix and metadata applied by :class:`BracketTracker`.
779 Trailing commas are maybe removed, unpacked for loop variables are
780 demoted from being delimiters.
782 Inline comments are put aside.
784 has_value = leaf.type in BRACKETS or bool(leaf.value.strip())
788 if token.COLON == leaf.type and self.is_class_paren_empty:
790 if self.leaves and not preformatted:
791 # Note: at this point leaf.prefix should be empty except for
792 # imports, for which we only preserve newlines.
793 leaf.prefix += whitespace(
794 leaf, complex_subscript=self.is_complex_subscript(leaf)
796 if self.inside_brackets or not preformatted:
797 self.bracket_tracker.mark(leaf)
798 self.maybe_remove_trailing_comma(leaf)
799 if not self.append_comment(leaf):
800 self.leaves.append(leaf)
802 def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
803 """Like :func:`append()` but disallow invalid standalone comment structure.
805 Raises ValueError when any `leaf` is appended after a standalone comment
806 or when a standalone comment is not the first leaf on the line.
808 if self.bracket_tracker.depth == 0:
810 raise ValueError("cannot append to standalone comments")
812 if self.leaves and leaf.type == STANDALONE_COMMENT:
814 "cannot append standalone comments to a populated line"
817 self.append(leaf, preformatted=preformatted)
820 def is_comment(self) -> bool:
821 """Is this line a standalone comment?"""
822 return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT
825 def is_decorator(self) -> bool:
826 """Is this line a decorator?"""
827 return bool(self) and self.leaves[0].type == token.AT
830 def is_import(self) -> bool:
831 """Is this an import line?"""
832 return bool(self) and is_import(self.leaves[0])
835 def is_class(self) -> bool:
836 """Is this line a class definition?"""
839 and self.leaves[0].type == token.NAME
840 and self.leaves[0].value == "class"
844 def is_trivial_class(self) -> bool:
845 """Is this line a class definition with a body consisting only of "..."?"""
848 and self.leaves[-3:] == [Leaf(token.DOT, ".") for _ in range(3)]
852 def is_def(self) -> bool:
853 """Is this a function definition? (Also returns True for async defs.)"""
855 first_leaf = self.leaves[0]
860 second_leaf: Optional[Leaf] = self.leaves[1]
864 (first_leaf.type == token.NAME and first_leaf.value == "def")
866 first_leaf.type == token.ASYNC
867 and second_leaf is not None
868 and second_leaf.type == token.NAME
869 and second_leaf.value == "def"
874 def is_flow_control(self) -> bool:
875 """Is this line a flow control statement?
877 Those are `return`, `raise`, `break`, and `continue`.
881 and self.leaves[0].type == token.NAME
882 and self.leaves[0].value in FLOW_CONTROL
886 def is_yield(self) -> bool:
887 """Is this line a yield statement?"""
890 and self.leaves[0].type == token.NAME
891 and self.leaves[0].value == "yield"
895 def is_class_paren_empty(self) -> bool:
896 """Is this a class with no base classes but using parentheses?
898 Those are unnecessary and should be removed.
902 and len(self.leaves) == 4
904 and self.leaves[2].type == token.LPAR
905 and self.leaves[2].value == "("
906 and self.leaves[3].type == token.RPAR
907 and self.leaves[3].value == ")"
910 def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
911 """If so, needs to be split before emitting."""
912 for leaf in self.leaves:
913 if leaf.type == STANDALONE_COMMENT:
914 if leaf.bracket_depth <= depth_limit:
919 def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
920 """Remove trailing comma if there is one and it's safe."""
923 and self.leaves[-1].type == token.COMMA
924 and closing.type in CLOSING_BRACKETS
928 if closing.type == token.RBRACE:
929 self.remove_trailing_comma()
932 if closing.type == token.RSQB:
933 comma = self.leaves[-1]
934 if comma.parent and comma.parent.type == syms.listmaker:
935 self.remove_trailing_comma()
938 # For parens let's check if it's safe to remove the comma.
939 # Imports are always safe.
941 self.remove_trailing_comma()
944 # Otheriwsse, if the trailing one is the only one, we might mistakenly
945 # change a tuple into a different type by removing the comma.
946 depth = closing.bracket_depth + 1
948 opening = closing.opening_bracket
949 for _opening_index, leaf in enumerate(self.leaves):
956 for leaf in self.leaves[_opening_index + 1 :]:
960 bracket_depth = leaf.bracket_depth
961 if bracket_depth == depth and leaf.type == token.COMMA:
963 if leaf.parent and leaf.parent.type == syms.arglist:
968 self.remove_trailing_comma()
973 def append_comment(self, comment: Leaf) -> bool:
974 """Add an inline or standalone comment to the line."""
976 comment.type == STANDALONE_COMMENT
977 and self.bracket_tracker.any_open_brackets()
982 if comment.type != token.COMMENT:
985 after = len(self.leaves) - 1
987 comment.type = STANDALONE_COMMENT
992 self.comments.append((after, comment))
995 def comments_after(self, leaf: Leaf, _index: int = -1) -> Iterator[Leaf]:
996 """Generate comments that should appear directly after `leaf`.
998 Provide a non-negative leaf `_index` to speed up the function.
1001 for _index, _leaf in enumerate(self.leaves):
1008 for index, comment_after in self.comments:
1012 def remove_trailing_comma(self) -> None:
1013 """Remove the trailing comma and moves the comments attached to it."""
1014 comma_index = len(self.leaves) - 1
1015 for i in range(len(self.comments)):
1016 comment_index, comment = self.comments[i]
1017 if comment_index == comma_index:
1018 self.comments[i] = (comma_index - 1, comment)
1021 def is_complex_subscript(self, leaf: Leaf) -> bool:
1022 """Return True iff `leaf` is part of a slice with non-trivial exprs."""
1024 leaf if leaf.type == token.LSQB else self.bracket_tracker.get_open_lsqb()
1026 if open_lsqb is None:
1029 subscript_start = open_lsqb.next_sibling
1031 isinstance(subscript_start, Node)
1032 and subscript_start.type == syms.subscriptlist
1034 subscript_start = child_towards(subscript_start, leaf)
1036 subscript_start is not None
1037 and any(n.type in TEST_DESCENDANTS for n in subscript_start.pre_order())
1040 def __str__(self) -> str:
1041 """Render the line."""
1045 indent = " " * self.depth
1046 leaves = iter(self.leaves)
1047 first = next(leaves)
1048 res = f"{first.prefix}{indent}{first.value}"
1051 for _, comment in self.comments:
1055 def __bool__(self) -> bool:
1056 """Return True if the line has leaves or comments."""
1057 return bool(self.leaves or self.comments)
1060 class UnformattedLines(Line):
1061 """Just like :class:`Line` but stores lines which aren't reformatted."""
1063 def append(self, leaf: Leaf, preformatted: bool = True) -> None:
1064 """Just add a new `leaf` to the end of the lines.
1066 The `preformatted` argument is ignored.
1068 Keeps track of indentation `depth`, which is useful when the user
1069 says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`.
1072 list(generate_comments(leaf))
1073 except FormatOn as f_on:
1074 self.leaves.append(f_on.leaf_from_consumed(leaf))
1077 self.leaves.append(leaf)
1078 if leaf.type == token.INDENT:
1080 elif leaf.type == token.DEDENT:
1083 def __str__(self) -> str:
1084 """Render unformatted lines from leaves which were added with `append()`.
1086 `depth` is not used for indentation in this case.
1092 for leaf in self.leaves:
1096 def append_comment(self, comment: Leaf) -> bool:
1097 """Not implemented in this class. Raises `NotImplementedError`."""
1098 raise NotImplementedError("Unformatted lines don't store comments separately.")
1100 def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
1101 """Does nothing and returns False."""
1104 def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
1105 """Does nothing and returns False."""
1110 class EmptyLineTracker:
1111 """Provides a stateful method that returns the number of potential extra
1112 empty lines needed before and after the currently processed line.
1114 Note: this tracker works on lines that haven't been split yet. It assumes
1115 the prefix of the first leaf consists of optional newlines. Those newlines
1116 are consumed by `maybe_empty_lines()` and included in the computation.
1118 is_pyi: bool = False
1119 previous_line: Optional[Line] = None
1120 previous_after: int = 0
1121 previous_defs: List[int] = Factory(list)
1123 def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1124 """Return the number of extra empty lines before and after the `current_line`.
1126 This is for separating `def`, `async def` and `class` with extra empty
1127 lines (two on module-level), as well as providing an extra empty line
1128 after flow control keywords to make them more prominent.
1130 if isinstance(current_line, UnformattedLines):
1133 before, after = self._maybe_empty_lines(current_line)
1134 before -= self.previous_after
1135 self.previous_after = after
1136 self.previous_line = current_line
1137 return before, after
1139 def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1141 if current_line.depth == 0:
1142 max_allowed = 1 if self.is_pyi else 2
1143 if current_line.leaves:
1144 # Consume the first leaf's extra newlines.
1145 first_leaf = current_line.leaves[0]
1146 before = first_leaf.prefix.count("\n")
1147 before = min(before, max_allowed)
1148 first_leaf.prefix = ""
1151 depth = current_line.depth
1152 while self.previous_defs and self.previous_defs[-1] >= depth:
1153 self.previous_defs.pop()
1155 before = 0 if depth else 1
1157 before = 1 if depth else 2
1158 is_decorator = current_line.is_decorator
1159 if is_decorator or current_line.is_def or current_line.is_class:
1160 if not is_decorator:
1161 self.previous_defs.append(depth)
1162 if self.previous_line is None:
1163 # Don't insert empty lines before the first line in the file.
1166 if self.previous_line.is_decorator:
1170 self.previous_line.is_comment
1171 and self.previous_line.depth == current_line.depth
1177 if self.previous_line.depth > current_line.depth:
1179 elif current_line.is_class or self.previous_line.is_class:
1181 current_line.is_trivial_class
1182 and self.previous_line.is_trivial_class
1191 if current_line.depth and newlines:
1197 and self.previous_line.is_import
1198 and not current_line.is_import
1199 and depth == self.previous_line.depth
1201 return (before or 1), 0
1207 class LineGenerator(Visitor[Line]):
1208 """Generates reformatted Line objects. Empty lines are not emitted.
1210 Note: destroys the tree it's visiting by mutating prefixes of its leaves
1211 in ways that will no longer stringify to valid Python code on the tree.
1213 is_pyi: bool = False
1214 current_line: Line = Factory(Line)
1215 remove_u_prefix: bool = False
1217 def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]:
1220 If the line is empty, only emit if it makes sense.
1221 If the line is too long, split it first and then generate.
1223 If any lines were generated, set up a new current_line.
1225 if not self.current_line:
1226 if self.current_line.__class__ == type:
1227 self.current_line.depth += indent
1229 self.current_line = type(depth=self.current_line.depth + indent)
1230 return # Line is empty, don't emit. Creating a new one unnecessary.
1232 complete_line = self.current_line
1233 self.current_line = type(depth=complete_line.depth + indent)
1236 def visit(self, node: LN) -> Iterator[Line]:
1237 """Main method to visit `node` and its children.
1239 Yields :class:`Line` objects.
1241 if isinstance(self.current_line, UnformattedLines):
1242 # File contained `# fmt: off`
1243 yield from self.visit_unformatted(node)
1246 yield from super().visit(node)
1248 def visit_default(self, node: LN) -> Iterator[Line]:
1249 """Default `visit_*()` implementation. Recurses to children of `node`."""
1250 if isinstance(node, Leaf):
1251 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
1253 for comment in generate_comments(node):
1254 if any_open_brackets:
1255 # any comment within brackets is subject to splitting
1256 self.current_line.append(comment)
1257 elif comment.type == token.COMMENT:
1258 # regular trailing comment
1259 self.current_line.append(comment)
1260 yield from self.line()
1263 # regular standalone comment
1264 yield from self.line()
1266 self.current_line.append(comment)
1267 yield from self.line()
1269 except FormatOff as f_off:
1270 f_off.trim_prefix(node)
1271 yield from self.line(type=UnformattedLines)
1272 yield from self.visit(node)
1274 except FormatOn as f_on:
1275 # This only happens here if somebody says "fmt: on" multiple
1277 f_on.trim_prefix(node)
1278 yield from self.visit_default(node)
1281 normalize_prefix(node, inside_brackets=any_open_brackets)
1282 if node.type == token.STRING:
1283 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
1284 normalize_string_quotes(node)
1285 if node.type not in WHITESPACE:
1286 self.current_line.append(node)
1287 yield from super().visit_default(node)
1289 def visit_INDENT(self, node: Node) -> Iterator[Line]:
1290 """Increase indentation level, maybe yield a line."""
1291 # In blib2to3 INDENT never holds comments.
1292 yield from self.line(+1)
1293 yield from self.visit_default(node)
1295 def visit_DEDENT(self, node: Node) -> Iterator[Line]:
1296 """Decrease indentation level, maybe yield a line."""
1297 # The current line might still wait for trailing comments. At DEDENT time
1298 # there won't be any (they would be prefixes on the preceding NEWLINE).
1299 # Emit the line then.
1300 yield from self.line()
1302 # While DEDENT has no value, its prefix may contain standalone comments
1303 # that belong to the current indentation level. Get 'em.
1304 yield from self.visit_default(node)
1306 # Finally, emit the dedent.
1307 yield from self.line(-1)
1310 self, node: Node, keywords: Set[str], parens: Set[str]
1311 ) -> Iterator[Line]:
1312 """Visit a statement.
1314 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
1315 `def`, `with`, `class`, `assert` and assignments.
1317 The relevant Python language `keywords` for a given statement will be
1318 NAME leaves within it. This methods puts those on a separate line.
1320 `parens` holds a set of string leaf values immeditely after which
1321 invisible parens should be put.
1323 normalize_invisible_parens(node, parens_after=parens)
1324 for child in node.children:
1325 if child.type == token.NAME and child.value in keywords: # type: ignore
1326 yield from self.line()
1328 yield from self.visit(child)
1330 def visit_suite(self, node: Node) -> Iterator[Line]:
1331 """Visit a suite."""
1332 if self.is_pyi and self.is_trivial_suite(node):
1333 yield from self.visit(node.children[2])
1335 yield from self.visit_default(node)
1337 def is_trivial_suite(self, node: Node) -> bool:
1338 if len(node.children) != 4:
1341 not isinstance(node.children[0], Leaf)
1342 or node.children[0].type != token.NEWLINE
1346 not isinstance(node.children[1], Leaf)
1347 or node.children[1].type != token.INDENT
1351 not isinstance(node.children[3], Leaf)
1352 or node.children[3].type != token.DEDENT
1355 stmt = node.children[2]
1356 if not isinstance(stmt, Node):
1358 return self.is_trivial_body(stmt)
1360 def is_trivial_body(self, stmt: Node) -> bool:
1361 if not isinstance(stmt, Node) or stmt.type != syms.simple_stmt:
1363 if len(stmt.children) != 2:
1365 child = stmt.children[0]
1367 child.type == syms.atom
1368 and len(child.children) == 3
1369 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
1372 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
1373 """Visit a statement without nested statements."""
1374 is_suite_like = node.parent and node.parent.type in STATEMENT
1376 if self.is_pyi and self.is_trivial_body(node):
1377 yield from self.visit_default(node)
1379 yield from self.line(+1)
1380 yield from self.visit_default(node)
1381 yield from self.line(-1)
1387 or not self.is_trivial_suite(node.parent)
1389 yield from self.line()
1390 yield from self.visit_default(node)
1392 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
1393 """Visit `async def`, `async for`, `async with`."""
1394 yield from self.line()
1396 children = iter(node.children)
1397 for child in children:
1398 yield from self.visit(child)
1400 if child.type == token.ASYNC:
1403 internal_stmt = next(children)
1404 for child in internal_stmt.children:
1405 yield from self.visit(child)
1407 def visit_decorators(self, node: Node) -> Iterator[Line]:
1408 """Visit decorators."""
1409 for child in node.children:
1410 yield from self.line()
1411 yield from self.visit(child)
1413 def visit_import_from(self, node: Node) -> Iterator[Line]:
1414 """Visit import_from and maybe put invisible parentheses.
1416 This is separate from `visit_stmt` because import statements don't
1417 support arbitrary atoms and thus handling of parentheses is custom.
1420 for index, child in enumerate(node.children):
1422 if child.type == token.LPAR:
1423 # make parentheses invisible
1424 child.value = "" # type: ignore
1425 node.children[-1].value = "" # type: ignore
1427 # insert invisible parentheses
1428 node.insert_child(index, Leaf(token.LPAR, ""))
1429 node.append_child(Leaf(token.RPAR, ""))
1433 child.type == token.NAME and child.value == "import" # type: ignore
1436 for child in node.children:
1437 yield from self.visit(child)
1439 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
1440 """Remove a semicolon and put the other statement on a separate line."""
1441 yield from self.line()
1443 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
1444 """End of file. Process outstanding comments and end with a newline."""
1445 yield from self.visit_default(leaf)
1446 yield from self.line()
1448 def visit_unformatted(self, node: LN) -> Iterator[Line]:
1449 """Used when file contained a `# fmt: off`."""
1450 if isinstance(node, Node):
1451 for child in node.children:
1452 yield from self.visit(child)
1456 self.current_line.append(node)
1457 except FormatOn as f_on:
1458 f_on.trim_prefix(node)
1459 yield from self.line()
1460 yield from self.visit(node)
1462 if node.type == token.ENDMARKER:
1463 # somebody decided not to put a final `# fmt: on`
1464 yield from self.line()
1466 def __attrs_post_init__(self) -> None:
1467 """You are in a twisty little maze of passages."""
1470 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
1471 self.visit_if_stmt = partial(
1472 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
1474 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
1475 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
1476 self.visit_try_stmt = partial(
1477 v, keywords={"try", "except", "else", "finally"}, parens=Ø
1479 self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
1480 self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
1481 self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
1482 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
1483 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
1484 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
1485 self.visit_async_funcdef = self.visit_async_stmt
1486 self.visit_decorated = self.visit_decorators
1489 IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist}
1490 BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE}
1491 OPENING_BRACKETS = set(BRACKET.keys())
1492 CLOSING_BRACKETS = set(BRACKET.values())
1493 BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS
1494 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
1497 def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa C901
1498 """Return whitespace prefix if needed for the given `leaf`.
1500 `complex_subscript` signals whether the given leaf is part of a subscription
1501 which has non-trivial arguments, like arithmetic expressions or function calls.
1509 if t in ALWAYS_NO_SPACE:
1512 if t == token.COMMENT:
1515 assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
1518 and p.type not in {syms.subscript, syms.subscriptlist, syms.sliceop}
1522 prev = leaf.prev_sibling
1524 prevp = preceding_leaf(p)
1525 if not prevp or prevp.type in OPENING_BRACKETS:
1528 if t == token.COLON:
1529 if prevp.type == token.COLON:
1532 elif prevp.type != token.COMMA and not complex_subscript:
1537 if prevp.type == token.EQUAL:
1539 if prevp.parent.type in {
1540 syms.arglist, syms.argument, syms.parameters, syms.varargslist
1544 elif prevp.parent.type == syms.typedargslist:
1545 # A bit hacky: if the equal sign has whitespace, it means we
1546 # previously found it's a typed argument. So, we're using
1550 elif prevp.type in STARS:
1551 if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1554 elif prevp.type == token.COLON:
1555 if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
1556 return SPACE if complex_subscript else NO
1560 and prevp.parent.type == syms.factor
1561 and prevp.type in MATH_OPERATORS
1566 prevp.type == token.RIGHTSHIFT
1568 and prevp.parent.type == syms.shift_expr
1569 and prevp.prev_sibling
1570 and prevp.prev_sibling.type == token.NAME
1571 and prevp.prev_sibling.value == "print" # type: ignore
1573 # Python 2 print chevron
1576 elif prev.type in OPENING_BRACKETS:
1579 if p.type in {syms.parameters, syms.arglist}:
1580 # untyped function signatures or calls
1581 if not prev or prev.type != token.COMMA:
1584 elif p.type == syms.varargslist:
1586 if prev and prev.type != token.COMMA:
1589 elif p.type == syms.typedargslist:
1590 # typed function signatures
1594 if t == token.EQUAL:
1595 if prev.type != syms.tname:
1598 elif prev.type == token.EQUAL:
1599 # A bit hacky: if the equal sign has whitespace, it means we
1600 # previously found it's a typed argument. So, we're using that, too.
1603 elif prev.type != token.COMMA:
1606 elif p.type == syms.tname:
1609 prevp = preceding_leaf(p)
1610 if not prevp or prevp.type != token.COMMA:
1613 elif p.type == syms.trailer:
1614 # attributes and calls
1615 if t == token.LPAR or t == token.RPAR:
1620 prevp = preceding_leaf(p)
1621 if not prevp or prevp.type != token.NUMBER:
1624 elif t == token.LSQB:
1627 elif prev.type != token.COMMA:
1630 elif p.type == syms.argument:
1632 if t == token.EQUAL:
1636 prevp = preceding_leaf(p)
1637 if not prevp or prevp.type == token.LPAR:
1640 elif prev.type in {token.EQUAL} | STARS:
1643 elif p.type == syms.decorator:
1647 elif p.type == syms.dotted_name:
1651 prevp = preceding_leaf(p)
1652 if not prevp or prevp.type == token.AT or prevp.type == token.DOT:
1655 elif p.type == syms.classdef:
1659 if prev and prev.type == token.LPAR:
1662 elif p.type in {syms.subscript, syms.sliceop}:
1665 assert p.parent is not None, "subscripts are always parented"
1666 if p.parent.type == syms.subscriptlist:
1671 elif not complex_subscript:
1674 elif p.type == syms.atom:
1675 if prev and t == token.DOT:
1676 # dots, but not the first one.
1679 elif p.type == syms.dictsetmaker:
1681 if prev and prev.type == token.DOUBLESTAR:
1684 elif p.type in {syms.factor, syms.star_expr}:
1687 prevp = preceding_leaf(p)
1688 if not prevp or prevp.type in OPENING_BRACKETS:
1691 prevp_parent = prevp.parent
1692 assert prevp_parent is not None
1694 prevp.type == token.COLON
1695 and prevp_parent.type in {syms.subscript, syms.sliceop}
1699 elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:
1702 elif t == token.NAME or t == token.NUMBER:
1705 elif p.type == syms.import_from:
1707 if prev and prev.type == token.DOT:
1710 elif t == token.NAME:
1714 if prev and prev.type == token.DOT:
1717 elif p.type == syms.sliceop:
1723 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
1724 """Return the first leaf that precedes `node`, if any."""
1726 res = node.prev_sibling
1728 if isinstance(res, Leaf):
1732 return list(res.leaves())[-1]
1741 def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
1742 """Return the child of `ancestor` that contains `descendant`."""
1743 node: Optional[LN] = descendant
1744 while node and node.parent != ancestor:
1749 def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
1750 """Return the priority of the `leaf` delimiter, given a line break after it.
1752 The delimiter priorities returned here are from those delimiters that would
1753 cause a line break after themselves.
1755 Higher numbers are higher priority.
1757 if leaf.type == token.COMMA:
1758 return COMMA_PRIORITY
1763 def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
1764 """Return the priority of the `leaf` delimiter, given a line before after it.
1766 The delimiter priorities returned here are from those delimiters that would
1767 cause a line break before themselves.
1769 Higher numbers are higher priority.
1771 if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1772 # * and ** might also be MATH_OPERATORS but in this case they are not.
1773 # Don't treat them as a delimiter.
1777 leaf.type in MATH_OPERATORS
1779 and leaf.parent.type not in {syms.factor, syms.star_expr}
1781 return MATH_PRIORITIES[leaf.type]
1783 if leaf.type in COMPARATORS:
1784 return COMPARATOR_PRIORITY
1787 leaf.type == token.STRING
1788 and previous is not None
1789 and previous.type == token.STRING
1791 return STRING_PRIORITY
1794 leaf.type == token.NAME
1795 and leaf.value == "for"
1797 and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
1799 return COMPREHENSION_PRIORITY
1802 leaf.type == token.NAME
1803 and leaf.value == "if"
1805 and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
1807 return COMPREHENSION_PRIORITY
1810 leaf.type == token.NAME
1811 and leaf.value in {"if", "else"}
1813 and leaf.parent.type == syms.test
1815 return TERNARY_PRIORITY
1817 if leaf.type == token.NAME and leaf.value in LOGIC_OPERATORS and leaf.parent:
1818 return LOGIC_PRIORITY
1823 def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
1824 """Clean the prefix of the `leaf` and generate comments from it, if any.
1826 Comments in lib2to3 are shoved into the whitespace prefix. This happens
1827 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
1828 move because it does away with modifying the grammar to include all the
1829 possible places in which comments can be placed.
1831 The sad consequence for us though is that comments don't "belong" anywhere.
1832 This is why this function generates simple parentless Leaf objects for
1833 comments. We simply don't know what the correct parent should be.
1835 No matter though, we can live without this. We really only need to
1836 differentiate between inline and standalone comments. The latter don't
1837 share the line with any code.
1839 Inline comments are emitted as regular token.COMMENT leaves. Standalone
1840 are emitted with a fake STANDALONE_COMMENT token identifier.
1851 for index, line in enumerate(p.split("\n")):
1852 consumed += len(line) + 1 # adding the length of the split '\n'
1853 line = line.lstrip()
1856 if not line.startswith("#"):
1859 if index == 0 and leaf.type != token.ENDMARKER:
1860 comment_type = token.COMMENT # simple trailing comment
1862 comment_type = STANDALONE_COMMENT
1863 comment = make_comment(line)
1864 yield Leaf(comment_type, comment, prefix="\n" * nlines)
1866 if comment in {"# fmt: on", "# yapf: enable"}:
1867 raise FormatOn(consumed)
1869 if comment in {"# fmt: off", "# yapf: disable"}:
1870 if comment_type == STANDALONE_COMMENT:
1871 raise FormatOff(consumed)
1873 prev = preceding_leaf(leaf)
1874 if not prev or prev.type in WHITESPACE: # standalone comment in disguise
1875 raise FormatOff(consumed)
1880 def make_comment(content: str) -> str:
1881 """Return a consistently formatted comment from the given `content` string.
1883 All comments (except for "##", "#!", "#:") should have a single space between
1884 the hash sign and the content.
1886 If `content` didn't start with a hash sign, one is provided.
1888 content = content.rstrip()
1892 if content[0] == "#":
1893 content = content[1:]
1894 if content and content[0] not in " !:#":
1895 content = " " + content
1896 return "#" + content
1900 line: Line, line_length: int, inner: bool = False, py36: bool = False
1901 ) -> Iterator[Line]:
1902 """Split a `line` into potentially many lines.
1904 They should fit in the allotted `line_length` but might not be able to.
1905 `inner` signifies that there were a pair of brackets somewhere around the
1906 current `line`, possibly transitively. This means we can fallback to splitting
1907 by delimiters if the LHS/RHS don't yield any results.
1909 If `py36` is True, splitting may generate syntax that is only compatible
1910 with Python 3.6 and later.
1912 if isinstance(line, UnformattedLines) or line.is_comment:
1916 line_str = str(line).strip("\n")
1917 if is_line_short_enough(line, line_length=line_length, line_str=line_str):
1921 split_funcs: List[SplitFunc]
1923 split_funcs = [left_hand_split]
1924 elif line.is_import:
1925 split_funcs = [explode_split]
1928 def rhs(line: Line, py36: bool = False) -> Iterator[Line]:
1929 for omit in generate_trailers_to_omit(line, line_length):
1930 lines = list(right_hand_split(line, py36, omit=omit))
1931 if is_line_short_enough(lines[0], line_length=line_length):
1935 # All splits failed, best effort split with no omits.
1936 yield from right_hand_split(line, py36)
1938 if line.inside_brackets:
1939 split_funcs = [delimiter_split, standalone_comment_split, rhs]
1942 for split_func in split_funcs:
1943 # We are accumulating lines in `result` because we might want to abort
1944 # mission and return the original line in the end, or attempt a different
1946 result: List[Line] = []
1948 for l in split_func(line, py36):
1949 if str(l).strip("\n") == line_str:
1950 raise CannotSplit("Split function returned an unchanged result")
1953 split_line(l, line_length=line_length, inner=True, py36=py36)
1955 except CannotSplit as cs:
1966 def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
1967 """Split line into many lines, starting with the first matching bracket pair.
1969 Note: this usually looks weird, only use this for function definitions.
1970 Prefer RHS otherwise. This is why this function is not symmetrical with
1971 :func:`right_hand_split` which also handles optional parentheses.
1973 head = Line(depth=line.depth)
1974 body = Line(depth=line.depth + 1, inside_brackets=True)
1975 tail = Line(depth=line.depth)
1976 tail_leaves: List[Leaf] = []
1977 body_leaves: List[Leaf] = []
1978 head_leaves: List[Leaf] = []
1979 current_leaves = head_leaves
1980 matching_bracket = None
1981 for leaf in line.leaves:
1983 current_leaves is body_leaves
1984 and leaf.type in CLOSING_BRACKETS
1985 and leaf.opening_bracket is matching_bracket
1987 current_leaves = tail_leaves if body_leaves else head_leaves
1988 current_leaves.append(leaf)
1989 if current_leaves is head_leaves:
1990 if leaf.type in OPENING_BRACKETS:
1991 matching_bracket = leaf
1992 current_leaves = body_leaves
1993 # Since body is a new indent level, remove spurious leading whitespace.
1995 normalize_prefix(body_leaves[0], inside_brackets=True)
1996 # Build the new lines.
1997 for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
1999 result.append(leaf, preformatted=True)
2000 for comment_after in line.comments_after(leaf):
2001 result.append(comment_after, preformatted=True)
2002 bracket_split_succeeded_or_raise(head, body, tail)
2003 for result in (head, body, tail):
2008 def right_hand_split(
2009 line: Line, py36: bool = False, omit: Collection[LeafID] = ()
2010 ) -> Iterator[Line]:
2011 """Split line into many lines, starting with the last matching bracket pair.
2013 If the split was by optional parentheses, attempt splitting without them, too.
2014 `omit` is a collection of closing bracket IDs that shouldn't be considered for
2017 Note: running this function modifies `bracket_depth` on the leaves of `line`.
2019 head = Line(depth=line.depth)
2020 body = Line(depth=line.depth + 1, inside_brackets=True)
2021 tail = Line(depth=line.depth)
2022 tail_leaves: List[Leaf] = []
2023 body_leaves: List[Leaf] = []
2024 head_leaves: List[Leaf] = []
2025 current_leaves = tail_leaves
2026 opening_bracket = None
2027 closing_bracket = None
2028 for leaf in reversed(line.leaves):
2029 if current_leaves is body_leaves:
2030 if leaf is opening_bracket:
2031 current_leaves = head_leaves if body_leaves else tail_leaves
2032 current_leaves.append(leaf)
2033 if current_leaves is tail_leaves:
2034 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
2035 opening_bracket = leaf.opening_bracket
2036 closing_bracket = leaf
2037 current_leaves = body_leaves
2038 tail_leaves.reverse()
2039 body_leaves.reverse()
2040 head_leaves.reverse()
2041 # Since body is a new indent level, remove spurious leading whitespace.
2043 normalize_prefix(body_leaves[0], inside_brackets=True)
2044 elif not head_leaves:
2045 # No `head` and no `body` means the split failed. `tail` has all content.
2046 raise CannotSplit("No brackets found")
2048 # Build the new lines.
2049 for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
2051 result.append(leaf, preformatted=True)
2052 for comment_after in line.comments_after(leaf):
2053 result.append(comment_after, preformatted=True)
2054 bracket_split_succeeded_or_raise(head, body, tail)
2055 assert opening_bracket and closing_bracket
2057 # the opening bracket is an optional paren
2058 opening_bracket.type == token.LPAR
2059 and not opening_bracket.value
2060 # the closing bracket is an optional paren
2061 and closing_bracket.type == token.RPAR
2062 and not closing_bracket.value
2063 # there are no delimiters or standalone comments in the body
2064 and not body.bracket_tracker.delimiters
2065 and not line.contains_standalone_comments(0)
2066 # and it's not an import (optional parens are the only thing we can split
2067 # on in this case; attempting a split without them is a waste of time)
2068 and not line.is_import
2070 omit = {id(closing_bracket), *omit}
2072 yield from right_hand_split(line, py36=py36, omit=omit)
2077 ensure_visible(opening_bracket)
2078 ensure_visible(closing_bracket)
2079 for result in (head, body, tail):
2084 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
2085 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
2087 Do nothing otherwise.
2089 A left- or right-hand split is based on a pair of brackets. Content before
2090 (and including) the opening bracket is left on one line, content inside the
2091 brackets is put on a separate line, and finally content starting with and
2092 following the closing bracket is put on a separate line.
2094 Those are called `head`, `body`, and `tail`, respectively. If the split
2095 produced the same line (all content in `head`) or ended up with an empty `body`
2096 and the `tail` is just the closing bracket, then it's considered failed.
2098 tail_len = len(str(tail).strip())
2101 raise CannotSplit("Splitting brackets produced the same line")
2105 f"Splitting brackets on an empty body to save "
2106 f"{tail_len} characters is not worth it"
2110 def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
2111 """Normalize prefix of the first leaf in every line returned by `split_func`.
2113 This is a decorator over relevant split functions.
2117 def split_wrapper(line: Line, py36: bool = False) -> Iterator[Line]:
2118 for l in split_func(line, py36):
2119 normalize_prefix(l.leaves[0], inside_brackets=True)
2122 return split_wrapper
2125 @dont_increase_indentation
2126 def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
2127 """Split according to delimiters of the highest priority.
2129 If `py36` is True, the split will add trailing commas also in function
2130 signatures that contain `*` and `**`.
2133 last_leaf = line.leaves[-1]
2135 raise CannotSplit("Line empty")
2137 delimiters = line.bracket_tracker.delimiters
2139 delimiter_priority = line.bracket_tracker.max_delimiter_priority(
2140 exclude={id(last_leaf)}
2143 raise CannotSplit("No delimiters found")
2145 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2146 lowest_depth = sys.maxsize
2147 trailing_comma_safe = True
2149 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2150 """Append `leaf` to current line or to new line if appending impossible."""
2151 nonlocal current_line
2153 current_line.append_safe(leaf, preformatted=True)
2154 except ValueError as ve:
2157 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2158 current_line.append(leaf)
2160 for index, leaf in enumerate(line.leaves):
2161 yield from append_to_line(leaf)
2163 for comment_after in line.comments_after(leaf, index):
2164 yield from append_to_line(comment_after)
2166 lowest_depth = min(lowest_depth, leaf.bracket_depth)
2168 leaf.bracket_depth == lowest_depth
2169 and is_vararg(leaf, within=VARARGS_PARENTS)
2171 trailing_comma_safe = trailing_comma_safe and py36
2172 leaf_priority = delimiters.get(id(leaf))
2173 if leaf_priority == delimiter_priority:
2176 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2180 and delimiter_priority == COMMA_PRIORITY
2181 and current_line.leaves[-1].type != token.COMMA
2182 and current_line.leaves[-1].type != STANDALONE_COMMENT
2184 current_line.append(Leaf(token.COMMA, ","))
2188 @dont_increase_indentation
2189 def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]:
2190 """Split standalone comments from the rest of the line."""
2191 if not line.contains_standalone_comments(0):
2192 raise CannotSplit("Line does not have any standalone comments")
2194 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2196 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2197 """Append `leaf` to current line or to new line if appending impossible."""
2198 nonlocal current_line
2200 current_line.append_safe(leaf, preformatted=True)
2201 except ValueError as ve:
2204 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2205 current_line.append(leaf)
2207 for index, leaf in enumerate(line.leaves):
2208 yield from append_to_line(leaf)
2210 for comment_after in line.comments_after(leaf, index):
2211 yield from append_to_line(comment_after)
2218 line: Line, py36: bool = False, omit: Collection[LeafID] = ()
2219 ) -> Iterator[Line]:
2220 """Split by rightmost bracket and immediately split contents by a delimiter."""
2221 new_lines = list(right_hand_split(line, py36, omit))
2222 if len(new_lines) != 3:
2223 yield from new_lines
2229 yield from delimiter_split(new_lines[1], py36)
2237 def is_import(leaf: Leaf) -> bool:
2238 """Return True if the given leaf starts an import statement."""
2245 (v == "import" and p and p.type == syms.import_name)
2246 or (v == "from" and p and p.type == syms.import_from)
2251 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
2252 """Leave existing extra newlines if not `inside_brackets`. Remove everything
2255 Note: don't use backslashes for formatting or you'll lose your voting rights.
2257 if not inside_brackets:
2258 spl = leaf.prefix.split("#")
2259 if "\\" not in spl[0]:
2260 nl_count = spl[-1].count("\n")
2263 leaf.prefix = "\n" * nl_count
2269 def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
2270 """Make all string prefixes lowercase.
2272 If remove_u_prefix is given, also removes any u prefix from the string.
2274 Note: Mutates its argument.
2276 match = re.match(r"^([furbFURB]*)(.*)$", leaf.value, re.DOTALL)
2277 assert match is not None, f"failed to match string {leaf.value!r}"
2278 orig_prefix = match.group(1)
2279 new_prefix = orig_prefix.lower()
2281 new_prefix = new_prefix.replace("u", "")
2282 leaf.value = f"{new_prefix}{match.group(2)}"
2285 def normalize_string_quotes(leaf: Leaf) -> None:
2286 """Prefer double quotes but only if it doesn't cause more escaping.
2288 Adds or removes backslashes as appropriate. Doesn't parse and fix
2289 strings nested in f-strings (yet).
2291 Note: Mutates its argument.
2293 value = leaf.value.lstrip("furbFURB")
2294 if value[:3] == '"""':
2297 elif value[:3] == "'''":
2300 elif value[0] == '"':
2306 first_quote_pos = leaf.value.find(orig_quote)
2307 if first_quote_pos == -1:
2308 return # There's an internal error
2310 prefix = leaf.value[:first_quote_pos]
2311 unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
2312 escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}")
2313 escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}")
2314 body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
2315 if "r" in prefix.casefold():
2316 if unescaped_new_quote.search(body):
2317 # There's at least one unescaped new_quote in this raw string
2318 # so converting is impossible
2321 # Do not introduce or remove backslashes in raw strings
2324 # remove unnecessary quotes
2325 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
2326 if body != new_body:
2327 # Consider the string without unnecessary quotes as the original
2329 leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}"
2330 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
2331 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
2332 if new_quote == '"""' and new_body[-1] == '"':
2334 new_body = new_body[:-1] + '\\"'
2335 orig_escape_count = body.count("\\")
2336 new_escape_count = new_body.count("\\")
2337 if new_escape_count > orig_escape_count:
2338 return # Do not introduce more escaping
2340 if new_escape_count == orig_escape_count and orig_quote == '"':
2341 return # Prefer double quotes
2343 leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
2346 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
2347 """Make existing optional parentheses invisible or create new ones.
2349 `parens_after` is a set of string leaf values immeditely after which parens
2352 Standardizes on visible parentheses for single-element tuples, and keeps
2353 existing visible parentheses for other tuples and generator expressions.
2356 for child in list(node.children):
2358 if child.type == syms.atom:
2359 maybe_make_parens_invisible_in_atom(child)
2360 elif is_one_tuple(child):
2361 # wrap child in visible parentheses
2362 lpar = Leaf(token.LPAR, "(")
2363 rpar = Leaf(token.RPAR, ")")
2364 index = child.remove() or 0
2365 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2367 # wrap child in invisible parentheses
2368 lpar = Leaf(token.LPAR, "")
2369 rpar = Leaf(token.RPAR, "")
2370 index = child.remove() or 0
2371 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2373 check_lpar = isinstance(child, Leaf) and child.value in parens_after
2376 def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
2377 """If it's safe, make the parens in the atom `node` invisible, recusively."""
2379 node.type != syms.atom
2380 or is_empty_tuple(node)
2381 or is_one_tuple(node)
2383 or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
2387 first = node.children[0]
2388 last = node.children[-1]
2389 if first.type == token.LPAR and last.type == token.RPAR:
2390 # make parentheses invisible
2391 first.value = "" # type: ignore
2392 last.value = "" # type: ignore
2393 if len(node.children) > 1:
2394 maybe_make_parens_invisible_in_atom(node.children[1])
2400 def is_empty_tuple(node: LN) -> bool:
2401 """Return True if `node` holds an empty tuple."""
2403 node.type == syms.atom
2404 and len(node.children) == 2
2405 and node.children[0].type == token.LPAR
2406 and node.children[1].type == token.RPAR
2410 def is_one_tuple(node: LN) -> bool:
2411 """Return True if `node` holds a tuple with one element, with or without parens."""
2412 if node.type == syms.atom:
2413 if len(node.children) != 3:
2416 lpar, gexp, rpar = node.children
2418 lpar.type == token.LPAR
2419 and gexp.type == syms.testlist_gexp
2420 and rpar.type == token.RPAR
2424 return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
2427 node.type in IMPLICIT_TUPLE
2428 and len(node.children) == 2
2429 and node.children[1].type == token.COMMA
2433 def is_yield(node: LN) -> bool:
2434 """Return True if `node` holds a `yield` or `yield from` expression."""
2435 if node.type == syms.yield_expr:
2438 if node.type == token.NAME and node.value == "yield": # type: ignore
2441 if node.type != syms.atom:
2444 if len(node.children) != 3:
2447 lpar, expr, rpar = node.children
2448 if lpar.type == token.LPAR and rpar.type == token.RPAR:
2449 return is_yield(expr)
2454 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
2455 """Return True if `leaf` is a star or double star in a vararg or kwarg.
2457 If `within` includes VARARGS_PARENTS, this applies to function signatures.
2458 If `within` includes UNPACKING_PARENTS, it applies to right hand-side
2459 extended iterable unpacking (PEP 3132) and additional unpacking
2460 generalizations (PEP 448).
2462 if leaf.type not in STARS or not leaf.parent:
2466 if p.type == syms.star_expr:
2467 # Star expressions are also used as assignment targets in extended
2468 # iterable unpacking (PEP 3132). See what its parent is instead.
2474 return p.type in within
2477 def max_delimiter_priority_in_atom(node: LN) -> int:
2478 """Return maximum delimiter priority inside `node`.
2480 This is specific to atoms with contents contained in a pair of parentheses.
2481 If `node` isn't an atom or there are no enclosing parentheses, returns 0.
2483 if node.type != syms.atom:
2486 first = node.children[0]
2487 last = node.children[-1]
2488 if not (first.type == token.LPAR and last.type == token.RPAR):
2491 bt = BracketTracker()
2492 for c in node.children[1:-1]:
2493 if isinstance(c, Leaf):
2496 for leaf in c.leaves():
2499 return bt.max_delimiter_priority()
2505 def ensure_visible(leaf: Leaf) -> None:
2506 """Make sure parentheses are visible.
2508 They could be invisible as part of some statements (see
2509 :func:`normalize_invible_parens` and :func:`visit_import_from`).
2511 if leaf.type == token.LPAR:
2513 elif leaf.type == token.RPAR:
2517 def is_python36(node: Node) -> bool:
2518 """Return True if the current file is using Python 3.6+ features.
2520 Currently looking for:
2522 - trailing commas after * or ** in function signatures and calls.
2524 for n in node.pre_order():
2525 if n.type == token.STRING:
2526 value_head = n.value[:2] # type: ignore
2527 if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
2531 n.type in {syms.typedargslist, syms.arglist}
2533 and n.children[-1].type == token.COMMA
2535 for ch in n.children:
2536 if ch.type in STARS:
2539 if ch.type == syms.argument:
2540 for argch in ch.children:
2541 if argch.type in STARS:
2547 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
2548 """Generate sets of closing bracket IDs that should be omitted in a RHS.
2550 Brackets can be omitted if the entire trailer up to and including
2551 a preceding closing bracket fits in one line.
2553 Yielded sets are cumulative (contain results of previous yields, too). First
2557 omit: Set[LeafID] = set()
2560 length = 4 * line.depth
2561 opening_bracket = None
2562 closing_bracket = None
2563 optional_brackets: Set[LeafID] = set()
2564 inner_brackets: Set[LeafID] = set()
2565 for index, leaf in enumerate_reversed(line.leaves):
2566 length += len(leaf.prefix) + len(leaf.value)
2567 if length > line_length:
2570 comment: Optional[Leaf]
2571 for comment in line.comments_after(leaf, index):
2572 if "\n" in comment.prefix:
2573 break # Oops, standalone comment!
2575 length += len(comment.value)
2578 if comment is not None:
2579 break # There was a standalone comment, we can't continue.
2581 optional_brackets.discard(id(leaf))
2583 if leaf is opening_bracket:
2584 opening_bracket = None
2585 elif leaf.type in CLOSING_BRACKETS:
2586 inner_brackets.add(id(leaf))
2587 elif leaf.type in CLOSING_BRACKETS:
2589 optional_brackets.add(id(opening_bracket))
2592 if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
2593 # Empty brackets would fail a split so treat them as "inner"
2594 # brackets (e.g. only add them to the `omit` set if another
2595 # pair of brackets was good enough.
2596 inner_brackets.add(id(leaf))
2599 opening_bracket = leaf.opening_bracket
2601 omit.add(id(closing_bracket))
2602 omit.update(inner_brackets)
2603 inner_brackets.clear()
2605 closing_bracket = leaf
2608 def get_future_imports(node: Node) -> Set[str]:
2609 """Return a set of __future__ imports in the file."""
2611 for child in node.children:
2612 if child.type != syms.simple_stmt:
2614 first_child = child.children[0]
2615 if isinstance(first_child, Leaf):
2616 # Continue looking if we see a docstring; otherwise stop.
2618 len(child.children) == 2
2619 and first_child.type == token.STRING
2620 and child.children[1].type == token.NEWLINE
2625 elif first_child.type == syms.import_from:
2626 module_name = first_child.children[1]
2627 if not isinstance(module_name, Leaf) or module_name.value != "__future__":
2629 for import_from_child in first_child.children[3:]:
2630 if isinstance(import_from_child, Leaf):
2631 if import_from_child.type == token.NAME:
2632 imports.add(import_from_child.value)
2634 assert import_from_child.type == syms.import_as_names
2635 for leaf in import_from_child.children:
2636 if isinstance(leaf, Leaf) and leaf.type == token.NAME:
2637 imports.add(leaf.value)
2643 PYTHON_EXTENSIONS = {".py", ".pyi"}
2644 BLACKLISTED_DIRECTORIES = {
2645 "build", "buck-out", "dist", "_build", ".git", ".hg", ".mypy_cache", ".tox", ".venv"
2649 def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
2650 """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
2651 and have one of the PYTHON_EXTENSIONS.
2653 for child in path.iterdir():
2655 if child.name in BLACKLISTED_DIRECTORIES:
2658 yield from gen_python_files_in_dir(child)
2660 elif child.is_file() and child.suffix in PYTHON_EXTENSIONS:
2666 """Provides a reformatting counter. Can be rendered with `str(report)`."""
2669 change_count: int = 0
2671 failure_count: int = 0
2673 def done(self, src: Path, changed: Changed) -> None:
2674 """Increment the counter for successful reformatting. Write out a message."""
2675 if changed is Changed.YES:
2676 reformatted = "would reformat" if self.check else "reformatted"
2678 out(f"{reformatted} {src}")
2679 self.change_count += 1
2682 if changed is Changed.NO:
2683 msg = f"{src} already well formatted, good job."
2685 msg = f"{src} wasn't modified on disk since last run."
2686 out(msg, bold=False)
2687 self.same_count += 1
2689 def failed(self, src: Path, message: str) -> None:
2690 """Increment the counter for failed reformatting. Write out a message."""
2691 err(f"error: cannot format {src}: {message}")
2692 self.failure_count += 1
2695 def return_code(self) -> int:
2696 """Return the exit code that the app should use.
2698 This considers the current state of changed files and failures:
2699 - if there were any failures, return 123;
2700 - if any files were changed and --check is being used, return 1;
2701 - otherwise return 0.
2703 # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
2704 # 126 we have special returncodes reserved by the shell.
2705 if self.failure_count:
2708 elif self.change_count and self.check:
2713 def __str__(self) -> str:
2714 """Render a color report of the current state.
2716 Use `click.unstyle` to remove colors.
2719 reformatted = "would be reformatted"
2720 unchanged = "would be left unchanged"
2721 failed = "would fail to reformat"
2723 reformatted = "reformatted"
2724 unchanged = "left unchanged"
2725 failed = "failed to reformat"
2727 if self.change_count:
2728 s = "s" if self.change_count > 1 else ""
2730 click.style(f"{self.change_count} file{s} {reformatted}", bold=True)
2733 s = "s" if self.same_count > 1 else ""
2734 report.append(f"{self.same_count} file{s} {unchanged}")
2735 if self.failure_count:
2736 s = "s" if self.failure_count > 1 else ""
2738 click.style(f"{self.failure_count} file{s} {failed}", fg="red")
2740 return ", ".join(report) + "."
2743 def assert_equivalent(src: str, dst: str) -> None:
2744 """Raise AssertionError if `src` and `dst` aren't equivalent."""
2749 def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
2750 """Simple visitor generating strings to compare ASTs by content."""
2751 yield f"{' ' * depth}{node.__class__.__name__}("
2753 for field in sorted(node._fields):
2755 value = getattr(node, field)
2756 except AttributeError:
2759 yield f"{' ' * (depth+1)}{field}="
2761 if isinstance(value, list):
2763 if isinstance(item, ast.AST):
2764 yield from _v(item, depth + 2)
2766 elif isinstance(value, ast.AST):
2767 yield from _v(value, depth + 2)
2770 yield f"{' ' * (depth+2)}{value!r}, # {value.__class__.__name__}"
2772 yield f"{' ' * depth}) # /{node.__class__.__name__}"
2775 src_ast = ast.parse(src)
2776 except Exception as exc:
2777 major, minor = sys.version_info[:2]
2778 raise AssertionError(
2779 f"cannot use --safe with this file; failed to parse source file "
2780 f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
2781 f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
2785 dst_ast = ast.parse(dst)
2786 except Exception as exc:
2787 log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
2788 raise AssertionError(
2789 f"INTERNAL ERROR: Black produced invalid code: {exc}. "
2790 f"Please report a bug on https://github.com/ambv/black/issues. "
2791 f"This invalid output might be helpful: {log}"
2794 src_ast_str = "\n".join(_v(src_ast))
2795 dst_ast_str = "\n".join(_v(dst_ast))
2796 if src_ast_str != dst_ast_str:
2797 log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
2798 raise AssertionError(
2799 f"INTERNAL ERROR: Black produced code that is not equivalent to "
2801 f"Please report a bug on https://github.com/ambv/black/issues. "
2802 f"This diff might be helpful: {log}"
2806 def assert_stable(src: str, dst: str, line_length: int, is_pyi: bool = False) -> None:
2807 """Raise AssertionError if `dst` reformats differently the second time."""
2808 newdst = format_str(dst, line_length=line_length, is_pyi=is_pyi)
2811 diff(src, dst, "source", "first pass"),
2812 diff(dst, newdst, "first pass", "second pass"),
2814 raise AssertionError(
2815 f"INTERNAL ERROR: Black produced different code on the second pass "
2816 f"of the formatter. "
2817 f"Please report a bug on https://github.com/ambv/black/issues. "
2818 f"This diff might be helpful: {log}"
2822 def dump_to_file(*output: str) -> str:
2823 """Dump `output` to a temporary file. Return path to the file."""
2826 with tempfile.NamedTemporaryFile(
2827 mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8"
2829 for lines in output:
2831 if lines and lines[-1] != "\n":
2836 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
2837 """Return a unified diff string between strings `a` and `b`."""
2840 a_lines = [line + "\n" for line in a.split("\n")]
2841 b_lines = [line + "\n" for line in b.split("\n")]
2843 difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5)
2847 def cancel(tasks: List[asyncio.Task]) -> None:
2848 """asyncio signal handler that cancels all `tasks` and reports to stderr."""
2854 def shutdown(loop: BaseEventLoop) -> None:
2855 """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
2857 # This part is borrowed from asyncio/runners.py in Python 3.7b2.
2858 to_cancel = [task for task in asyncio.Task.all_tasks(loop) if not task.done()]
2862 for task in to_cancel:
2864 loop.run_until_complete(
2865 asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)
2868 # `concurrent.futures.Future` objects cannot be cancelled once they
2869 # are already running. There might be some when the `shutdown()` happened.
2870 # Silence their logger's spew about the event loop being closed.
2871 cf_logger = logging.getLogger("concurrent.futures")
2872 cf_logger.setLevel(logging.CRITICAL)
2876 def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
2877 """Replace `regex` with `replacement` twice on `original`.
2879 This is used by string normalization to perform replaces on
2880 overlapping matches.
2882 return regex.sub(replacement, regex.sub(replacement, original))
2885 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
2886 """Like `reversed(enumerate(sequence))` if that were possible."""
2887 index = len(sequence) - 1
2888 for element in reversed(sequence):
2889 yield (index, element)
2893 def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
2894 """Return True if `line` is no longer than `line_length`.
2896 Uses the provided `line_str` rendering, if any, otherwise computes a new one.
2899 line_str = str(line).strip("\n")
2901 len(line_str) <= line_length
2902 and "\n" not in line_str # multiline strings
2903 and not line.contains_standalone_comments()
2907 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
2910 def get_cache_file(line_length: int) -> Path:
2911 return CACHE_DIR / f"cache.{line_length}.pickle"
2914 def read_cache(line_length: int) -> Cache:
2915 """Read the cache if it exists and is well formed.
2917 If it is not well formed, the call to write_cache later should resolve the issue.
2919 cache_file = get_cache_file(line_length)
2920 if not cache_file.exists():
2923 with cache_file.open("rb") as fobj:
2925 cache: Cache = pickle.load(fobj)
2926 except pickle.UnpicklingError:
2932 def get_cache_info(path: Path) -> CacheInfo:
2933 """Return the information used to check if a file is already formatted or not."""
2935 return stat.st_mtime, stat.st_size
2939 cache: Cache, sources: Iterable[Path]
2940 ) -> Tuple[List[Path], List[Path]]:
2941 """Split a list of paths into two.
2943 The first list contains paths of files that modified on disk or are not in the
2944 cache. The other list contains paths to non-modified files.
2949 if cache.get(src) != get_cache_info(src):
2956 def write_cache(cache: Cache, sources: List[Path], line_length: int) -> None:
2957 """Update the cache file."""
2958 cache_file = get_cache_file(line_length)
2960 if not CACHE_DIR.exists():
2961 CACHE_DIR.mkdir(parents=True)
2962 new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
2963 with cache_file.open("wb") as fobj:
2964 pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL)
2969 if __name__ == "__main__":