All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
3 from asyncio.base_events import BaseEventLoop
4 from concurrent.futures import Executor, ProcessPoolExecutor
6 from functools import partial, wraps
9 from multiprocessing import Manager
11 from pathlib import Path
35 from appdirs import user_cache_dir
36 from attr import dataclass, Factory
40 from blib2to3.pytree import Node, Leaf, type_repr
41 from blib2to3 import pygram, pytree
42 from blib2to3.pgen2 import driver, token
43 from blib2to3.pgen2.parse import ParseError
46 __version__ = "18.4a6"
47 DEFAULT_LINE_LENGTH = 88
50 syms = pygram.python_symbols
58 LN = Union[Leaf, Node]
59 SplitFunc = Callable[["Line", bool], Iterator["Line"]]
62 CacheInfo = Tuple[Timestamp, FileSize]
63 Cache = Dict[Path, CacheInfo]
64 out = partial(click.secho, bold=True, err=True)
65 err = partial(click.secho, fg="red", err=True)
68 class NothingChanged(UserWarning):
69 """Raised by :func:`format_file` when reformatted code is the same as source."""
72 class CannotSplit(Exception):
73 """A readable split that fits the allotted line length is impossible.
75 Raised by :func:`left_hand_split`, :func:`right_hand_split`, and
76 :func:`delimiter_split`.
80 class FormatError(Exception):
81 """Base exception for `# fmt: on` and `# fmt: off` handling.
83 It holds the number of bytes of the prefix consumed before the format
84 control comment appeared.
87 def __init__(self, consumed: int) -> None:
88 super().__init__(consumed)
89 self.consumed = consumed
91 def trim_prefix(self, leaf: Leaf) -> None:
92 leaf.prefix = leaf.prefix[self.consumed :]
94 def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
95 """Returns a new Leaf from the consumed part of the prefix."""
96 unformatted_prefix = leaf.prefix[: self.consumed]
97 return Leaf(token.NEWLINE, unformatted_prefix)
100 class FormatOn(FormatError):
101 """Found a comment like `# fmt: on` in the file."""
104 class FormatOff(FormatError):
105 """Found a comment like `# fmt: off` in the file."""
108 class WriteBack(Enum):
125 default=DEFAULT_LINE_LENGTH,
126 help="How many character per line to allow.",
133 "Don't write the files back, just return the status. Return code 0 "
134 "means nothing would change. Return code 1 means some files would be "
135 "reformatted. Return code 123 means there was an internal error."
141 help="Don't write the files back, just output a diff for each file on stdout.",
146 help="If --fast given, skip temporary sanity checks. [default: --safe]",
153 "Don't emit non-error messages to stderr. Errors are still emitted, "
154 "silence those with 2>/dev/null."
157 @click.version_option(version=__version__)
162 exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
175 """The uncompromising code formatter."""
176 sources: List[Path] = []
180 sources.extend(gen_python_files_in_dir(p))
182 # if a file was explicitly given, we don't care about its extension
185 sources.append(Path("-"))
187 err(f"invalid path: {s}")
189 if check and not diff:
190 write_back = WriteBack.NO
192 write_back = WriteBack.DIFF
194 write_back = WriteBack.YES
195 report = Report(check=check, quiet=quiet)
196 if len(sources) == 0:
197 out("No paths given. Nothing to do 😴")
201 elif len(sources) == 1:
202 reformat_one(sources[0], line_length, fast, write_back, report)
204 loop = asyncio.get_event_loop()
205 executor = ProcessPoolExecutor(max_workers=os.cpu_count())
207 loop.run_until_complete(
209 sources, line_length, fast, write_back, report, loop, executor
215 out("All done! ✨ 🍰 ✨")
216 click.echo(str(report))
217 ctx.exit(report.return_code)
221 src: Path, line_length: int, fast: bool, write_back: WriteBack, report: "Report"
223 """Reformat a single file under `src` without spawning child processes.
225 If `quiet` is True, non-error messages are not output. `line_length`,
226 `write_back`, and `fast` options are passed to :func:`format_file_in_place`.
230 if not src.is_file() and str(src) == "-":
231 if format_stdin_to_stdout(
232 line_length=line_length, fast=fast, write_back=write_back
234 changed = Changed.YES
237 if write_back != WriteBack.DIFF:
238 cache = read_cache(line_length)
240 if src in cache and cache[src] == get_cache_info(src):
241 changed = Changed.CACHED
243 changed is not Changed.CACHED
244 and format_file_in_place(
245 src, line_length=line_length, fast=fast, write_back=write_back
248 changed = Changed.YES
249 if write_back == WriteBack.YES and changed is not Changed.NO:
250 write_cache(cache, [src], line_length)
251 report.done(src, changed)
252 except Exception as exc:
253 report.failed(src, str(exc))
256 async def schedule_formatting(
260 write_back: WriteBack,
265 """Run formatting of `sources` in parallel using the provided `executor`.
267 (Use ProcessPoolExecutors for actual parallelism.)
269 `line_length`, `write_back`, and `fast` options are passed to
270 :func:`format_file_in_place`.
273 if write_back != WriteBack.DIFF:
274 cache = read_cache(line_length)
275 sources, cached = filter_cached(cache, sources)
277 report.done(src, Changed.CACHED)
282 if write_back == WriteBack.DIFF:
283 # For diff output, we need locks to ensure we don't interleave output
284 # from different processes.
286 lock = manager.Lock()
288 src: loop.run_in_executor(
289 executor, format_file_in_place, src, line_length, fast, write_back, lock
293 _task_values = list(tasks.values())
295 loop.add_signal_handler(signal.SIGINT, cancel, _task_values)
296 loop.add_signal_handler(signal.SIGTERM, cancel, _task_values)
297 except NotImplementedError:
298 # There are no good alternatives for these on Windows
300 await asyncio.wait(_task_values)
301 for src, task in tasks.items():
303 report.failed(src, "timed out, cancelling")
305 cancelled.append(task)
306 elif task.cancelled():
307 cancelled.append(task)
308 elif task.exception():
309 report.failed(src, str(task.exception()))
311 formatted.append(src)
312 report.done(src, Changed.YES if task.result() else Changed.NO)
315 await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
316 if write_back == WriteBack.YES and formatted:
317 write_cache(cache, formatted, line_length)
320 def format_file_in_place(
324 write_back: WriteBack = WriteBack.NO,
325 lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy
327 """Format file under `src` path. Return True if changed.
329 If `write_back` is True, write reformatted code back to stdout.
330 `line_length` and `fast` options are passed to :func:`format_file_contents`.
332 is_pyi = src.suffix == ".pyi"
334 with tokenize.open(src) as src_buffer:
335 src_contents = src_buffer.read()
337 dst_contents = format_file_contents(
338 src_contents, line_length=line_length, fast=fast, is_pyi=is_pyi
340 except NothingChanged:
343 if write_back == write_back.YES:
344 with open(src, "w", encoding=src_buffer.encoding) as f:
345 f.write(dst_contents)
346 elif write_back == write_back.DIFF:
347 src_name = f"{src} (original)"
348 dst_name = f"{src} (formatted)"
349 diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
353 sys.stdout.write(diff_contents)
360 def format_stdin_to_stdout(
361 line_length: int, fast: bool, write_back: WriteBack = WriteBack.NO
363 """Format file on stdin. Return True if changed.
365 If `write_back` is True, write reformatted code back to stdout.
366 `line_length` and `fast` arguments are passed to :func:`format_file_contents`.
368 src = sys.stdin.read()
371 dst = format_file_contents(src, line_length=line_length, fast=fast)
374 except NothingChanged:
378 if write_back == WriteBack.YES:
379 sys.stdout.write(dst)
380 elif write_back == WriteBack.DIFF:
381 src_name = "<stdin> (original)"
382 dst_name = "<stdin> (formatted)"
383 sys.stdout.write(diff(src, dst, src_name, dst_name))
386 def format_file_contents(
387 src_contents: str, *, line_length: int, fast: bool, is_pyi: bool = False
389 """Reformat contents a file and return new contents.
391 If `fast` is False, additionally confirm that the reformatted code is
392 valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
393 `line_length` is passed to :func:`format_str`.
395 if src_contents.strip() == "":
398 dst_contents = format_str(src_contents, line_length=line_length, is_pyi=is_pyi)
399 if src_contents == dst_contents:
403 assert_equivalent(src_contents, dst_contents)
405 src_contents, dst_contents, line_length=line_length, is_pyi=is_pyi
411 src_contents: str, line_length: int, *, is_pyi: bool = False
413 """Reformat a string and return new contents.
415 `line_length` determines how many characters per line are allowed.
417 src_node = lib2to3_parse(src_contents)
419 future_imports = get_future_imports(src_node)
420 elt = EmptyLineTracker(is_pyi=is_pyi)
421 py36 = is_python36(src_node)
422 lines = LineGenerator(
423 remove_u_prefix=py36 or "unicode_literals" in future_imports, is_pyi=is_pyi
427 for current_line in lines.visit(src_node):
428 for _ in range(after):
429 dst_contents += str(empty_line)
430 before, after = elt.maybe_empty_lines(current_line)
431 for _ in range(before):
432 dst_contents += str(empty_line)
433 for line in split_line(current_line, line_length=line_length, py36=py36):
434 dst_contents += str(line)
439 pygram.python_grammar_no_print_statement_no_exec_statement,
440 pygram.python_grammar_no_print_statement,
441 pygram.python_grammar,
445 def lib2to3_parse(src_txt: str) -> Node:
446 """Given a string with source, return the lib2to3 Node."""
447 grammar = pygram.python_grammar_no_print_statement
448 if src_txt[-1] != "\n":
449 nl = "\r\n" if "\r\n" in src_txt[:1024] else "\n"
451 for grammar in GRAMMARS:
452 drv = driver.Driver(grammar, pytree.convert)
454 result = drv.parse_string(src_txt, True)
457 except ParseError as pe:
458 lineno, column = pe.context[1]
459 lines = src_txt.splitlines()
461 faulty_line = lines[lineno - 1]
463 faulty_line = "<line number missing in source>"
464 exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
468 if isinstance(result, Leaf):
469 result = Node(syms.file_input, [result])
473 def lib2to3_unparse(node: Node) -> str:
474 """Given a lib2to3 node, return its string representation."""
482 class Visitor(Generic[T]):
483 """Basic lib2to3 visitor that yields things of type `T` on `visit()`."""
485 def visit(self, node: LN) -> Iterator[T]:
486 """Main method to visit `node` and its children.
488 It tries to find a `visit_*()` method for the given `node.type`, like
489 `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.
490 If no dedicated `visit_*()` method is found, chooses `visit_default()`
493 Then yields objects of type `T` from the selected visitor.
496 name = token.tok_name[node.type]
498 name = type_repr(node.type)
499 yield from getattr(self, f"visit_{name}", self.visit_default)(node)
501 def visit_default(self, node: LN) -> Iterator[T]:
502 """Default `visit_*()` implementation. Recurses to children of `node`."""
503 if isinstance(node, Node):
504 for child in node.children:
505 yield from self.visit(child)
509 class DebugVisitor(Visitor[T]):
512 def visit_default(self, node: LN) -> Iterator[T]:
513 indent = " " * (2 * self.tree_depth)
514 if isinstance(node, Node):
515 _type = type_repr(node.type)
516 out(f"{indent}{_type}", fg="yellow")
518 for child in node.children:
519 yield from self.visit(child)
522 out(f"{indent}/{_type}", fg="yellow", bold=False)
524 _type = token.tok_name.get(node.type, str(node.type))
525 out(f"{indent}{_type}", fg="blue", nl=False)
527 # We don't have to handle prefixes for `Node` objects since
528 # that delegates to the first child anyway.
529 out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
530 out(f" {node.value!r}", fg="blue", bold=False)
533 def show(cls, code: str) -> None:
534 """Pretty-print the lib2to3 AST of a given string of `code`.
536 Convenience method for debugging.
538 v: DebugVisitor[None] = DebugVisitor()
539 list(v.visit(lib2to3_parse(code)))
542 KEYWORDS = set(keyword.kwlist)
543 WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
544 FLOW_CONTROL = {"return", "raise", "break", "continue"}
555 STANDALONE_COMMENT = 153
556 LOGIC_OPERATORS = {"and", "or"}
581 STARS = {token.STAR, token.DOUBLESTAR}
584 syms.argument, # double star in arglist
585 syms.trailer, # single argument to call
587 syms.varargslist, # lambdas
589 UNPACKING_PARENTS = {
590 syms.atom, # single element of a list or set literal
628 COMPREHENSION_PRIORITY = 20
630 TERNARY_PRIORITY = 16
633 COMPARATOR_PRIORITY = 10
644 token.DOUBLESLASH: 3,
653 class BracketTracker:
654 """Keeps track of brackets on a line."""
657 bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
658 delimiters: Dict[LeafID, Priority] = Factory(dict)
659 previous: Optional[Leaf] = None
660 _for_loop_variable: int = 0
661 _lambda_arguments: int = 0
663 def mark(self, leaf: Leaf) -> None:
664 """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
666 All leaves receive an int `bracket_depth` field that stores how deep
667 within brackets a given leaf is. 0 means there are no enclosing brackets
668 that started on this line.
670 If a leaf is itself a closing bracket, it receives an `opening_bracket`
671 field that it forms a pair with. This is a one-directional link to
672 avoid reference cycles.
674 If a leaf is a delimiter (a token on which Black can split the line if
675 needed) and it's on depth 0, its `id()` is stored in the tracker's
678 if leaf.type == token.COMMENT:
681 self.maybe_decrement_after_for_loop_variable(leaf)
682 self.maybe_decrement_after_lambda_arguments(leaf)
683 if leaf.type in CLOSING_BRACKETS:
685 opening_bracket = self.bracket_match.pop((self.depth, leaf.type))
686 leaf.opening_bracket = opening_bracket
687 leaf.bracket_depth = self.depth
689 delim = is_split_before_delimiter(leaf, self.previous)
690 if delim and self.previous is not None:
691 self.delimiters[id(self.previous)] = delim
693 delim = is_split_after_delimiter(leaf, self.previous)
695 self.delimiters[id(leaf)] = delim
696 if leaf.type in OPENING_BRACKETS:
697 self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
700 self.maybe_increment_lambda_arguments(leaf)
701 self.maybe_increment_for_loop_variable(leaf)
703 def any_open_brackets(self) -> bool:
704 """Return True if there is an yet unmatched open bracket on the line."""
705 return bool(self.bracket_match)
707 def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
708 """Return the highest priority of a delimiter found on the line.
710 Values are consistent with what `is_split_*_delimiter()` return.
711 Raises ValueError on no delimiters.
713 return max(v for k, v in self.delimiters.items() if k not in exclude)
715 def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
716 """In a for loop, or comprehension, the variables are often unpacks.
718 To avoid splitting on the comma in this situation, increase the depth of
719 tokens between `for` and `in`.
721 if leaf.type == token.NAME and leaf.value == "for":
723 self._for_loop_variable += 1
728 def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
729 """See `maybe_increment_for_loop_variable` above for explanation."""
730 if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
732 self._for_loop_variable -= 1
737 def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool:
738 """In a lambda expression, there might be more than one argument.
740 To avoid splitting on the comma in this situation, increase the depth of
741 tokens between `lambda` and `:`.
743 if leaf.type == token.NAME and leaf.value == "lambda":
745 self._lambda_arguments += 1
750 def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
751 """See `maybe_increment_lambda_arguments` above for explanation."""
752 if self._lambda_arguments and leaf.type == token.COLON:
754 self._lambda_arguments -= 1
759 def get_open_lsqb(self) -> Optional[Leaf]:
760 """Return the most recent opening square bracket (if any)."""
761 return self.bracket_match.get((self.depth - 1, token.RSQB))
766 """Holds leaves and comments. Can be printed with `str(line)`."""
769 leaves: List[Leaf] = Factory(list)
770 comments: List[Tuple[Index, Leaf]] = Factory(list)
771 bracket_tracker: BracketTracker = Factory(BracketTracker)
772 inside_brackets: bool = False
774 def append(self, leaf: Leaf, preformatted: bool = False) -> None:
775 """Add a new `leaf` to the end of the line.
777 Unless `preformatted` is True, the `leaf` will receive a new consistent
778 whitespace prefix and metadata applied by :class:`BracketTracker`.
779 Trailing commas are maybe removed, unpacked for loop variables are
780 demoted from being delimiters.
782 Inline comments are put aside.
784 has_value = leaf.type in BRACKETS or bool(leaf.value.strip())
788 if token.COLON == leaf.type and self.is_class_paren_empty:
790 if self.leaves and not preformatted:
791 # Note: at this point leaf.prefix should be empty except for
792 # imports, for which we only preserve newlines.
793 leaf.prefix += whitespace(
794 leaf, complex_subscript=self.is_complex_subscript(leaf)
796 if self.inside_brackets or not preformatted:
797 self.bracket_tracker.mark(leaf)
798 self.maybe_remove_trailing_comma(leaf)
799 if not self.append_comment(leaf):
800 self.leaves.append(leaf)
802 def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
803 """Like :func:`append()` but disallow invalid standalone comment structure.
805 Raises ValueError when any `leaf` is appended after a standalone comment
806 or when a standalone comment is not the first leaf on the line.
808 if self.bracket_tracker.depth == 0:
810 raise ValueError("cannot append to standalone comments")
812 if self.leaves and leaf.type == STANDALONE_COMMENT:
814 "cannot append standalone comments to a populated line"
817 self.append(leaf, preformatted=preformatted)
820 def is_comment(self) -> bool:
821 """Is this line a standalone comment?"""
822 return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT
825 def is_decorator(self) -> bool:
826 """Is this line a decorator?"""
827 return bool(self) and self.leaves[0].type == token.AT
830 def is_import(self) -> bool:
831 """Is this an import line?"""
832 return bool(self) and is_import(self.leaves[0])
835 def is_class(self) -> bool:
836 """Is this line a class definition?"""
839 and self.leaves[0].type == token.NAME
840 and self.leaves[0].value == "class"
844 def is_trivial_class(self) -> bool:
845 """Is this line a class definition with a body consisting only of "..."?"""
848 and self.leaves[-3:] == [Leaf(token.DOT, ".") for _ in range(3)]
852 def is_def(self) -> bool:
853 """Is this a function definition? (Also returns True for async defs.)"""
855 first_leaf = self.leaves[0]
860 second_leaf: Optional[Leaf] = self.leaves[1]
864 (first_leaf.type == token.NAME and first_leaf.value == "def")
866 first_leaf.type == token.ASYNC
867 and second_leaf is not None
868 and second_leaf.type == token.NAME
869 and second_leaf.value == "def"
874 def is_flow_control(self) -> bool:
875 """Is this line a flow control statement?
877 Those are `return`, `raise`, `break`, and `continue`.
881 and self.leaves[0].type == token.NAME
882 and self.leaves[0].value in FLOW_CONTROL
886 def is_yield(self) -> bool:
887 """Is this line a yield statement?"""
890 and self.leaves[0].type == token.NAME
891 and self.leaves[0].value == "yield"
895 def is_class_paren_empty(self) -> bool:
896 """Is this a class with no base classes but using parentheses?
898 Those are unnecessary and should be removed.
902 and len(self.leaves) == 4
904 and self.leaves[2].type == token.LPAR
905 and self.leaves[2].value == "("
906 and self.leaves[3].type == token.RPAR
907 and self.leaves[3].value == ")"
910 def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
911 """If so, needs to be split before emitting."""
912 for leaf in self.leaves:
913 if leaf.type == STANDALONE_COMMENT:
914 if leaf.bracket_depth <= depth_limit:
919 def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
920 """Remove trailing comma if there is one and it's safe."""
923 and self.leaves[-1].type == token.COMMA
924 and closing.type in CLOSING_BRACKETS
928 if closing.type == token.RBRACE:
929 self.remove_trailing_comma()
932 if closing.type == token.RSQB:
933 comma = self.leaves[-1]
934 if comma.parent and comma.parent.type == syms.listmaker:
935 self.remove_trailing_comma()
938 # For parens let's check if it's safe to remove the comma.
939 # Imports are always safe.
941 self.remove_trailing_comma()
944 # Otheriwsse, if the trailing one is the only one, we might mistakenly
945 # change a tuple into a different type by removing the comma.
946 depth = closing.bracket_depth + 1
948 opening = closing.opening_bracket
949 for _opening_index, leaf in enumerate(self.leaves):
956 for leaf in self.leaves[_opening_index + 1 :]:
960 bracket_depth = leaf.bracket_depth
961 if bracket_depth == depth and leaf.type == token.COMMA:
963 if leaf.parent and leaf.parent.type == syms.arglist:
968 self.remove_trailing_comma()
973 def append_comment(self, comment: Leaf) -> bool:
974 """Add an inline or standalone comment to the line."""
976 comment.type == STANDALONE_COMMENT
977 and self.bracket_tracker.any_open_brackets()
982 if comment.type != token.COMMENT:
985 after = len(self.leaves) - 1
987 comment.type = STANDALONE_COMMENT
992 self.comments.append((after, comment))
995 def comments_after(self, leaf: Leaf, _index: int = -1) -> Iterator[Leaf]:
996 """Generate comments that should appear directly after `leaf`.
998 Provide a non-negative leaf `_index` to speed up the function.
1001 for _index, _leaf in enumerate(self.leaves):
1008 for index, comment_after in self.comments:
1012 def remove_trailing_comma(self) -> None:
1013 """Remove the trailing comma and moves the comments attached to it."""
1014 comma_index = len(self.leaves) - 1
1015 for i in range(len(self.comments)):
1016 comment_index, comment = self.comments[i]
1017 if comment_index == comma_index:
1018 self.comments[i] = (comma_index - 1, comment)
1021 def is_complex_subscript(self, leaf: Leaf) -> bool:
1022 """Return True iff `leaf` is part of a slice with non-trivial exprs."""
1024 leaf if leaf.type == token.LSQB else self.bracket_tracker.get_open_lsqb()
1026 if open_lsqb is None:
1029 subscript_start = open_lsqb.next_sibling
1031 isinstance(subscript_start, Node)
1032 and subscript_start.type == syms.subscriptlist
1034 subscript_start = child_towards(subscript_start, leaf)
1036 subscript_start is not None
1037 and any(n.type in TEST_DESCENDANTS for n in subscript_start.pre_order())
1040 def __str__(self) -> str:
1041 """Render the line."""
1045 indent = " " * self.depth
1046 leaves = iter(self.leaves)
1047 first = next(leaves)
1048 res = f"{first.prefix}{indent}{first.value}"
1051 for _, comment in self.comments:
1055 def __bool__(self) -> bool:
1056 """Return True if the line has leaves or comments."""
1057 return bool(self.leaves or self.comments)
1060 class UnformattedLines(Line):
1061 """Just like :class:`Line` but stores lines which aren't reformatted."""
1063 def append(self, leaf: Leaf, preformatted: bool = True) -> None:
1064 """Just add a new `leaf` to the end of the lines.
1066 The `preformatted` argument is ignored.
1068 Keeps track of indentation `depth`, which is useful when the user
1069 says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`.
1072 list(generate_comments(leaf))
1073 except FormatOn as f_on:
1074 self.leaves.append(f_on.leaf_from_consumed(leaf))
1077 self.leaves.append(leaf)
1078 if leaf.type == token.INDENT:
1080 elif leaf.type == token.DEDENT:
1083 def __str__(self) -> str:
1084 """Render unformatted lines from leaves which were added with `append()`.
1086 `depth` is not used for indentation in this case.
1092 for leaf in self.leaves:
1096 def append_comment(self, comment: Leaf) -> bool:
1097 """Not implemented in this class. Raises `NotImplementedError`."""
1098 raise NotImplementedError("Unformatted lines don't store comments separately.")
1100 def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
1101 """Does nothing and returns False."""
1104 def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
1105 """Does nothing and returns False."""
1110 class EmptyLineTracker:
1111 """Provides a stateful method that returns the number of potential extra
1112 empty lines needed before and after the currently processed line.
1114 Note: this tracker works on lines that haven't been split yet. It assumes
1115 the prefix of the first leaf consists of optional newlines. Those newlines
1116 are consumed by `maybe_empty_lines()` and included in the computation.
1118 is_pyi: bool = False
1119 previous_line: Optional[Line] = None
1120 previous_after: int = 0
1121 previous_defs: List[int] = Factory(list)
1123 def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1124 """Return the number of extra empty lines before and after the `current_line`.
1126 This is for separating `def`, `async def` and `class` with extra empty
1127 lines (two on module-level), as well as providing an extra empty line
1128 after flow control keywords to make them more prominent.
1130 if isinstance(current_line, UnformattedLines):
1133 before, after = self._maybe_empty_lines(current_line)
1134 before -= self.previous_after
1135 self.previous_after = after
1136 self.previous_line = current_line
1137 return before, after
1139 def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1141 if current_line.depth == 0:
1142 max_allowed = 1 if self.is_pyi else 2
1143 if current_line.leaves:
1144 # Consume the first leaf's extra newlines.
1145 first_leaf = current_line.leaves[0]
1146 before = first_leaf.prefix.count("\n")
1147 before = min(before, max_allowed)
1148 first_leaf.prefix = ""
1151 depth = current_line.depth
1152 while self.previous_defs and self.previous_defs[-1] >= depth:
1153 self.previous_defs.pop()
1155 before = 0 if depth else 1
1157 before = 1 if depth else 2
1158 is_decorator = current_line.is_decorator
1159 if is_decorator or current_line.is_def or current_line.is_class:
1160 if not is_decorator:
1161 self.previous_defs.append(depth)
1162 if self.previous_line is None:
1163 # Don't insert empty lines before the first line in the file.
1166 if self.previous_line.is_decorator:
1170 self.previous_line.is_comment
1171 and self.previous_line.depth == current_line.depth
1177 if self.previous_line.depth > current_line.depth:
1179 elif current_line.is_class or self.previous_line.is_class:
1181 current_line.is_trivial_class
1182 and self.previous_line.is_trivial_class
1191 if current_line.depth and newlines:
1197 and self.previous_line.is_import
1198 and not current_line.is_import
1199 and depth == self.previous_line.depth
1201 return (before or 1), 0
1207 class LineGenerator(Visitor[Line]):
1208 """Generates reformatted Line objects. Empty lines are not emitted.
1210 Note: destroys the tree it's visiting by mutating prefixes of its leaves
1211 in ways that will no longer stringify to valid Python code on the tree.
1213 is_pyi: bool = False
1214 current_line: Line = Factory(Line)
1215 remove_u_prefix: bool = False
1217 def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]:
1220 If the line is empty, only emit if it makes sense.
1221 If the line is too long, split it first and then generate.
1223 If any lines were generated, set up a new current_line.
1225 if not self.current_line:
1226 if self.current_line.__class__ == type:
1227 self.current_line.depth += indent
1229 self.current_line = type(depth=self.current_line.depth + indent)
1230 return # Line is empty, don't emit. Creating a new one unnecessary.
1232 complete_line = self.current_line
1233 self.current_line = type(depth=complete_line.depth + indent)
1236 def visit(self, node: LN) -> Iterator[Line]:
1237 """Main method to visit `node` and its children.
1239 Yields :class:`Line` objects.
1241 if isinstance(self.current_line, UnformattedLines):
1242 # File contained `# fmt: off`
1243 yield from self.visit_unformatted(node)
1246 yield from super().visit(node)
1248 def visit_default(self, node: LN) -> Iterator[Line]:
1249 """Default `visit_*()` implementation. Recurses to children of `node`."""
1250 if isinstance(node, Leaf):
1251 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
1253 for comment in generate_comments(node):
1254 if any_open_brackets:
1255 # any comment within brackets is subject to splitting
1256 self.current_line.append(comment)
1257 elif comment.type == token.COMMENT:
1258 # regular trailing comment
1259 self.current_line.append(comment)
1260 yield from self.line()
1263 # regular standalone comment
1264 yield from self.line()
1266 self.current_line.append(comment)
1267 yield from self.line()
1269 except FormatOff as f_off:
1270 f_off.trim_prefix(node)
1271 yield from self.line(type=UnformattedLines)
1272 yield from self.visit(node)
1274 except FormatOn as f_on:
1275 # This only happens here if somebody says "fmt: on" multiple
1277 f_on.trim_prefix(node)
1278 yield from self.visit_default(node)
1281 normalize_prefix(node, inside_brackets=any_open_brackets)
1282 if node.type == token.STRING:
1283 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
1284 normalize_string_quotes(node)
1285 if node.type not in WHITESPACE:
1286 self.current_line.append(node)
1287 yield from super().visit_default(node)
1289 def visit_INDENT(self, node: Node) -> Iterator[Line]:
1290 """Increase indentation level, maybe yield a line."""
1291 # In blib2to3 INDENT never holds comments.
1292 yield from self.line(+1)
1293 yield from self.visit_default(node)
1295 def visit_DEDENT(self, node: Node) -> Iterator[Line]:
1296 """Decrease indentation level, maybe yield a line."""
1297 # The current line might still wait for trailing comments. At DEDENT time
1298 # there won't be any (they would be prefixes on the preceding NEWLINE).
1299 # Emit the line then.
1300 yield from self.line()
1302 # While DEDENT has no value, its prefix may contain standalone comments
1303 # that belong to the current indentation level. Get 'em.
1304 yield from self.visit_default(node)
1306 # Finally, emit the dedent.
1307 yield from self.line(-1)
1310 self, node: Node, keywords: Set[str], parens: Set[str]
1311 ) -> Iterator[Line]:
1312 """Visit a statement.
1314 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
1315 `def`, `with`, `class`, `assert` and assignments.
1317 The relevant Python language `keywords` for a given statement will be
1318 NAME leaves within it. This methods puts those on a separate line.
1320 `parens` holds a set of string leaf values immeditely after which
1321 invisible parens should be put.
1323 normalize_invisible_parens(node, parens_after=parens)
1324 for child in node.children:
1325 if child.type == token.NAME and child.value in keywords: # type: ignore
1326 yield from self.line()
1328 yield from self.visit(child)
1330 def visit_suite(self, node: Node) -> Iterator[Line]:
1331 """Visit a suite."""
1332 if self.is_pyi and self.is_trivial_suite(node):
1333 yield from self.visit(node.children[2])
1335 yield from self.visit_default(node)
1337 def is_trivial_suite(self, node: Node) -> bool:
1338 if len(node.children) != 4:
1341 not isinstance(node.children[0], Leaf)
1342 or node.children[0].type != token.NEWLINE
1346 not isinstance(node.children[1], Leaf)
1347 or node.children[1].type != token.INDENT
1351 not isinstance(node.children[3], Leaf)
1352 or node.children[3].type != token.DEDENT
1355 stmt = node.children[2]
1356 if not isinstance(stmt, Node):
1358 return self.is_trivial_body(stmt)
1360 def is_trivial_body(self, stmt: Node) -> bool:
1361 if not isinstance(stmt, Node) or stmt.type != syms.simple_stmt:
1363 if len(stmt.children) != 2:
1365 child = stmt.children[0]
1367 child.type == syms.atom
1368 and len(child.children) == 3
1369 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
1372 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
1373 """Visit a statement without nested statements."""
1374 is_suite_like = node.parent and node.parent.type in STATEMENT
1376 if self.is_pyi and self.is_trivial_body(node):
1377 yield from self.visit_default(node)
1379 yield from self.line(+1)
1380 yield from self.visit_default(node)
1381 yield from self.line(-1)
1387 or not self.is_trivial_suite(node.parent)
1389 yield from self.line()
1390 yield from self.visit_default(node)
1392 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
1393 """Visit `async def`, `async for`, `async with`."""
1394 yield from self.line()
1396 children = iter(node.children)
1397 for child in children:
1398 yield from self.visit(child)
1400 if child.type == token.ASYNC:
1403 internal_stmt = next(children)
1404 for child in internal_stmt.children:
1405 yield from self.visit(child)
1407 def visit_decorators(self, node: Node) -> Iterator[Line]:
1408 """Visit decorators."""
1409 for child in node.children:
1410 yield from self.line()
1411 yield from self.visit(child)
1413 def visit_import_from(self, node: Node) -> Iterator[Line]:
1414 """Visit import_from and maybe put invisible parentheses.
1416 This is separate from `visit_stmt` because import statements don't
1417 support arbitrary atoms and thus handling of parentheses is custom.
1420 for index, child in enumerate(node.children):
1422 if child.type == token.LPAR:
1423 # make parentheses invisible
1424 child.value = "" # type: ignore
1425 node.children[-1].value = "" # type: ignore
1427 # insert invisible parentheses
1428 node.insert_child(index, Leaf(token.LPAR, ""))
1429 node.append_child(Leaf(token.RPAR, ""))
1433 child.type == token.NAME and child.value == "import" # type: ignore
1436 for child in node.children:
1437 yield from self.visit(child)
1439 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
1440 """Remove a semicolon and put the other statement on a separate line."""
1441 yield from self.line()
1443 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
1444 """End of file. Process outstanding comments and end with a newline."""
1445 yield from self.visit_default(leaf)
1446 yield from self.line()
1448 def visit_unformatted(self, node: LN) -> Iterator[Line]:
1449 """Used when file contained a `# fmt: off`."""
1450 if isinstance(node, Node):
1451 for child in node.children:
1452 yield from self.visit(child)
1456 self.current_line.append(node)
1457 except FormatOn as f_on:
1458 f_on.trim_prefix(node)
1459 yield from self.line()
1460 yield from self.visit(node)
1462 if node.type == token.ENDMARKER:
1463 # somebody decided not to put a final `# fmt: on`
1464 yield from self.line()
1466 def __attrs_post_init__(self) -> None:
1467 """You are in a twisty little maze of passages."""
1470 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
1471 self.visit_if_stmt = partial(
1472 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
1474 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
1475 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
1476 self.visit_try_stmt = partial(
1477 v, keywords={"try", "except", "else", "finally"}, parens=Ø
1479 self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
1480 self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
1481 self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
1482 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
1483 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
1484 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
1485 self.visit_async_funcdef = self.visit_async_stmt
1486 self.visit_decorated = self.visit_decorators
1489 IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist}
1490 BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE}
1491 OPENING_BRACKETS = set(BRACKET.keys())
1492 CLOSING_BRACKETS = set(BRACKET.values())
1493 BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS
1494 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
1497 def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa C901
1498 """Return whitespace prefix if needed for the given `leaf`.
1500 `complex_subscript` signals whether the given leaf is part of a subscription
1501 which has non-trivial arguments, like arithmetic expressions or function calls.
1509 if t in ALWAYS_NO_SPACE:
1512 if t == token.COMMENT:
1515 assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
1518 and p.type not in {syms.subscript, syms.subscriptlist, syms.sliceop}
1522 prev = leaf.prev_sibling
1524 prevp = preceding_leaf(p)
1525 if not prevp or prevp.type in OPENING_BRACKETS:
1528 if t == token.COLON:
1529 if prevp.type == token.COLON:
1532 elif prevp.type != token.COMMA and not complex_subscript:
1537 if prevp.type == token.EQUAL:
1539 if prevp.parent.type in {
1540 syms.arglist, syms.argument, syms.parameters, syms.varargslist
1544 elif prevp.parent.type == syms.typedargslist:
1545 # A bit hacky: if the equal sign has whitespace, it means we
1546 # previously found it's a typed argument. So, we're using
1550 elif prevp.type in STARS:
1551 if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1554 elif prevp.type == token.COLON:
1555 if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
1556 return SPACE if complex_subscript else NO
1560 and prevp.parent.type == syms.factor
1561 and prevp.type in MATH_OPERATORS
1566 prevp.type == token.RIGHTSHIFT
1568 and prevp.parent.type == syms.shift_expr
1569 and prevp.prev_sibling
1570 and prevp.prev_sibling.type == token.NAME
1571 and prevp.prev_sibling.value == "print" # type: ignore
1573 # Python 2 print chevron
1576 elif prev.type in OPENING_BRACKETS:
1579 if p.type in {syms.parameters, syms.arglist}:
1580 # untyped function signatures or calls
1581 if not prev or prev.type != token.COMMA:
1584 elif p.type == syms.varargslist:
1586 if prev and prev.type != token.COMMA:
1589 elif p.type == syms.typedargslist:
1590 # typed function signatures
1594 if t == token.EQUAL:
1595 if prev.type != syms.tname:
1598 elif prev.type == token.EQUAL:
1599 # A bit hacky: if the equal sign has whitespace, it means we
1600 # previously found it's a typed argument. So, we're using that, too.
1603 elif prev.type != token.COMMA:
1606 elif p.type == syms.tname:
1609 prevp = preceding_leaf(p)
1610 if not prevp or prevp.type != token.COMMA:
1613 elif p.type == syms.trailer:
1614 # attributes and calls
1615 if t == token.LPAR or t == token.RPAR:
1620 prevp = preceding_leaf(p)
1621 if not prevp or prevp.type != token.NUMBER:
1624 elif t == token.LSQB:
1627 elif prev.type != token.COMMA:
1630 elif p.type == syms.argument:
1632 if t == token.EQUAL:
1636 prevp = preceding_leaf(p)
1637 if not prevp or prevp.type == token.LPAR:
1640 elif prev.type in {token.EQUAL} | STARS:
1643 elif p.type == syms.decorator:
1647 elif p.type == syms.dotted_name:
1651 prevp = preceding_leaf(p)
1652 if not prevp or prevp.type == token.AT or prevp.type == token.DOT:
1655 elif p.type == syms.classdef:
1659 if prev and prev.type == token.LPAR:
1662 elif p.type in {syms.subscript, syms.sliceop}:
1665 assert p.parent is not None, "subscripts are always parented"
1666 if p.parent.type == syms.subscriptlist:
1671 elif not complex_subscript:
1674 elif p.type == syms.atom:
1675 if prev and t == token.DOT:
1676 # dots, but not the first one.
1679 elif p.type == syms.dictsetmaker:
1681 if prev and prev.type == token.DOUBLESTAR:
1684 elif p.type in {syms.factor, syms.star_expr}:
1687 prevp = preceding_leaf(p)
1688 if not prevp or prevp.type in OPENING_BRACKETS:
1691 prevp_parent = prevp.parent
1692 assert prevp_parent is not None
1694 prevp.type == token.COLON
1695 and prevp_parent.type in {syms.subscript, syms.sliceop}
1699 elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:
1702 elif t == token.NAME or t == token.NUMBER:
1705 elif p.type == syms.import_from:
1707 if prev and prev.type == token.DOT:
1710 elif t == token.NAME:
1714 if prev and prev.type == token.DOT:
1717 elif p.type == syms.sliceop:
1723 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
1724 """Return the first leaf that precedes `node`, if any."""
1726 res = node.prev_sibling
1728 if isinstance(res, Leaf):
1732 return list(res.leaves())[-1]
1741 def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
1742 """Return the child of `ancestor` that contains `descendant`."""
1743 node: Optional[LN] = descendant
1744 while node and node.parent != ancestor:
1749 def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
1750 """Return the priority of the `leaf` delimiter, given a line break after it.
1752 The delimiter priorities returned here are from those delimiters that would
1753 cause a line break after themselves.
1755 Higher numbers are higher priority.
1757 if leaf.type == token.COMMA:
1758 return COMMA_PRIORITY
1763 def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
1764 """Return the priority of the `leaf` delimiter, given a line before after it.
1766 The delimiter priorities returned here are from those delimiters that would
1767 cause a line break before themselves.
1769 Higher numbers are higher priority.
1771 if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1772 # * and ** might also be MATH_OPERATORS but in this case they are not.
1773 # Don't treat them as a delimiter.
1777 leaf.type in MATH_OPERATORS
1779 and leaf.parent.type not in {syms.factor, syms.star_expr}
1781 return MATH_PRIORITIES[leaf.type]
1783 if leaf.type in COMPARATORS:
1784 return COMPARATOR_PRIORITY
1787 leaf.type == token.STRING
1788 and previous is not None
1789 and previous.type == token.STRING
1791 return STRING_PRIORITY
1794 leaf.type == token.NAME
1795 and leaf.value == "for"
1797 and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
1799 return COMPREHENSION_PRIORITY
1802 leaf.type == token.NAME
1803 and leaf.value == "if"
1805 and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
1807 return COMPREHENSION_PRIORITY
1810 leaf.type == token.NAME
1811 and leaf.value in {"if", "else"}
1813 and leaf.parent.type == syms.test
1815 return TERNARY_PRIORITY
1817 if leaf.type == token.NAME and leaf.value in LOGIC_OPERATORS and leaf.parent:
1818 return LOGIC_PRIORITY
1823 def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
1824 """Clean the prefix of the `leaf` and generate comments from it, if any.
1826 Comments in lib2to3 are shoved into the whitespace prefix. This happens
1827 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
1828 move because it does away with modifying the grammar to include all the
1829 possible places in which comments can be placed.
1831 The sad consequence for us though is that comments don't "belong" anywhere.
1832 This is why this function generates simple parentless Leaf objects for
1833 comments. We simply don't know what the correct parent should be.
1835 No matter though, we can live without this. We really only need to
1836 differentiate between inline and standalone comments. The latter don't
1837 share the line with any code.
1839 Inline comments are emitted as regular token.COMMENT leaves. Standalone
1840 are emitted with a fake STANDALONE_COMMENT token identifier.
1851 for index, line in enumerate(p.split("\n")):
1852 consumed += len(line) + 1 # adding the length of the split '\n'
1853 line = line.lstrip()
1856 if not line.startswith("#"):
1859 if index == 0 and leaf.type != token.ENDMARKER:
1860 comment_type = token.COMMENT # simple trailing comment
1862 comment_type = STANDALONE_COMMENT
1863 comment = make_comment(line)
1864 yield Leaf(comment_type, comment, prefix="\n" * nlines)
1866 if comment in {"# fmt: on", "# yapf: enable"}:
1867 raise FormatOn(consumed)
1869 if comment in {"# fmt: off", "# yapf: disable"}:
1870 if comment_type == STANDALONE_COMMENT:
1871 raise FormatOff(consumed)
1873 prev = preceding_leaf(leaf)
1874 if not prev or prev.type in WHITESPACE: # standalone comment in disguise
1875 raise FormatOff(consumed)
1880 def make_comment(content: str) -> str:
1881 """Return a consistently formatted comment from the given `content` string.
1883 All comments (except for "##", "#!", "#:") should have a single space between
1884 the hash sign and the content.
1886 If `content` didn't start with a hash sign, one is provided.
1888 content = content.rstrip()
1892 if content[0] == "#":
1893 content = content[1:]
1894 if content and content[0] not in " !:#":
1895 content = " " + content
1896 return "#" + content
1900 line: Line, line_length: int, inner: bool = False, py36: bool = False
1901 ) -> Iterator[Line]:
1902 """Split a `line` into potentially many lines.
1904 They should fit in the allotted `line_length` but might not be able to.
1905 `inner` signifies that there were a pair of brackets somewhere around the
1906 current `line`, possibly transitively. This means we can fallback to splitting
1907 by delimiters if the LHS/RHS don't yield any results.
1909 If `py36` is True, splitting may generate syntax that is only compatible
1910 with Python 3.6 and later.
1912 if isinstance(line, UnformattedLines) or line.is_comment:
1916 line_str = str(line).strip("\n")
1917 if is_line_short_enough(line, line_length=line_length, line_str=line_str):
1921 split_funcs: List[SplitFunc]
1923 split_funcs = [left_hand_split]
1924 elif line.is_import:
1925 split_funcs = [explode_split]
1928 def rhs(line: Line, py36: bool = False) -> Iterator[Line]:
1929 for omit in generate_trailers_to_omit(line, line_length):
1930 lines = list(right_hand_split(line, py36, omit=omit))
1931 if is_line_short_enough(lines[0], line_length=line_length):
1935 # All splits failed, best effort split with no omits.
1936 yield from right_hand_split(line, py36)
1938 if line.inside_brackets:
1939 split_funcs = [delimiter_split, standalone_comment_split, rhs]
1942 for split_func in split_funcs:
1943 # We are accumulating lines in `result` because we might want to abort
1944 # mission and return the original line in the end, or attempt a different
1946 result: List[Line] = []
1948 for l in split_func(line, py36):
1949 if str(l).strip("\n") == line_str:
1950 raise CannotSplit("Split function returned an unchanged result")
1953 split_line(l, line_length=line_length, inner=True, py36=py36)
1955 except CannotSplit as cs:
1966 def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
1967 """Split line into many lines, starting with the first matching bracket pair.
1969 Note: this usually looks weird, only use this for function definitions.
1970 Prefer RHS otherwise. This is why this function is not symmetrical with
1971 :func:`right_hand_split` which also handles optional parentheses.
1973 head = Line(depth=line.depth)
1974 body = Line(depth=line.depth + 1, inside_brackets=True)
1975 tail = Line(depth=line.depth)
1976 tail_leaves: List[Leaf] = []
1977 body_leaves: List[Leaf] = []
1978 head_leaves: List[Leaf] = []
1979 current_leaves = head_leaves
1980 matching_bracket = None
1981 for leaf in line.leaves:
1983 current_leaves is body_leaves
1984 and leaf.type in CLOSING_BRACKETS
1985 and leaf.opening_bracket is matching_bracket
1987 current_leaves = tail_leaves if body_leaves else head_leaves
1988 current_leaves.append(leaf)
1989 if current_leaves is head_leaves:
1990 if leaf.type in OPENING_BRACKETS:
1991 matching_bracket = leaf
1992 current_leaves = body_leaves
1993 # Since body is a new indent level, remove spurious leading whitespace.
1995 normalize_prefix(body_leaves[0], inside_brackets=True)
1996 # Build the new lines.
1997 for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
1999 result.append(leaf, preformatted=True)
2000 for comment_after in line.comments_after(leaf):
2001 result.append(comment_after, preformatted=True)
2002 bracket_split_succeeded_or_raise(head, body, tail)
2003 for result in (head, body, tail):
2008 def right_hand_split(
2009 line: Line, py36: bool = False, omit: Collection[LeafID] = ()
2010 ) -> Iterator[Line]:
2011 """Split line into many lines, starting with the last matching bracket pair.
2013 If the split was by optional parentheses, attempt splitting without them, too.
2014 `omit` is a collection of closing bracket IDs that shouldn't be considered for
2017 head = Line(depth=line.depth)
2018 body = Line(depth=line.depth + 1, inside_brackets=True)
2019 tail = Line(depth=line.depth)
2020 tail_leaves: List[Leaf] = []
2021 body_leaves: List[Leaf] = []
2022 head_leaves: List[Leaf] = []
2023 current_leaves = tail_leaves
2024 opening_bracket = None
2025 closing_bracket = None
2026 for leaf in reversed(line.leaves):
2027 if current_leaves is body_leaves:
2028 if leaf is opening_bracket:
2029 current_leaves = head_leaves if body_leaves else tail_leaves
2030 current_leaves.append(leaf)
2031 if current_leaves is tail_leaves:
2032 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
2033 opening_bracket = leaf.opening_bracket
2034 closing_bracket = leaf
2035 current_leaves = body_leaves
2036 tail_leaves.reverse()
2037 body_leaves.reverse()
2038 head_leaves.reverse()
2039 # Since body is a new indent level, remove spurious leading whitespace.
2041 normalize_prefix(body_leaves[0], inside_brackets=True)
2042 elif not head_leaves:
2043 # No `head` and no `body` means the split failed. `tail` has all content.
2044 raise CannotSplit("No brackets found")
2046 # Build the new lines.
2047 for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
2049 result.append(leaf, preformatted=True)
2050 for comment_after in line.comments_after(leaf):
2051 result.append(comment_after, preformatted=True)
2052 bracket_split_succeeded_or_raise(head, body, tail)
2053 assert opening_bracket and closing_bracket
2055 # the opening bracket is an optional paren
2056 opening_bracket.type == token.LPAR
2057 and not opening_bracket.value
2058 # the closing bracket is an optional paren
2059 and closing_bracket.type == token.RPAR
2060 and not closing_bracket.value
2061 # there are no delimiters or standalone comments in the body
2062 and not body.bracket_tracker.delimiters
2063 and not line.contains_standalone_comments(0)
2064 # and it's not an import (optional parens are the only thing we can split
2065 # on in this case; attempting a split without them is a waste of time)
2066 and not line.is_import
2068 omit = {id(closing_bracket), *omit}
2070 yield from right_hand_split(line, py36=py36, omit=omit)
2075 ensure_visible(opening_bracket)
2076 ensure_visible(closing_bracket)
2077 for result in (head, body, tail):
2082 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
2083 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
2085 Do nothing otherwise.
2087 A left- or right-hand split is based on a pair of brackets. Content before
2088 (and including) the opening bracket is left on one line, content inside the
2089 brackets is put on a separate line, and finally content starting with and
2090 following the closing bracket is put on a separate line.
2092 Those are called `head`, `body`, and `tail`, respectively. If the split
2093 produced the same line (all content in `head`) or ended up with an empty `body`
2094 and the `tail` is just the closing bracket, then it's considered failed.
2096 tail_len = len(str(tail).strip())
2099 raise CannotSplit("Splitting brackets produced the same line")
2103 f"Splitting brackets on an empty body to save "
2104 f"{tail_len} characters is not worth it"
2108 def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
2109 """Normalize prefix of the first leaf in every line returned by `split_func`.
2111 This is a decorator over relevant split functions.
2115 def split_wrapper(line: Line, py36: bool = False) -> Iterator[Line]:
2116 for l in split_func(line, py36):
2117 normalize_prefix(l.leaves[0], inside_brackets=True)
2120 return split_wrapper
2123 @dont_increase_indentation
2124 def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
2125 """Split according to delimiters of the highest priority.
2127 If `py36` is True, the split will add trailing commas also in function
2128 signatures that contain `*` and `**`.
2131 last_leaf = line.leaves[-1]
2133 raise CannotSplit("Line empty")
2135 delimiters = line.bracket_tracker.delimiters
2137 delimiter_priority = line.bracket_tracker.max_delimiter_priority(
2138 exclude={id(last_leaf)}
2141 raise CannotSplit("No delimiters found")
2143 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2144 lowest_depth = sys.maxsize
2145 trailing_comma_safe = True
2147 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2148 """Append `leaf` to current line or to new line if appending impossible."""
2149 nonlocal current_line
2151 current_line.append_safe(leaf, preformatted=True)
2152 except ValueError as ve:
2155 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2156 current_line.append(leaf)
2158 for index, leaf in enumerate(line.leaves):
2159 yield from append_to_line(leaf)
2161 for comment_after in line.comments_after(leaf, index):
2162 yield from append_to_line(comment_after)
2164 lowest_depth = min(lowest_depth, leaf.bracket_depth)
2166 leaf.bracket_depth == lowest_depth
2167 and is_vararg(leaf, within=VARARGS_PARENTS)
2169 trailing_comma_safe = trailing_comma_safe and py36
2170 leaf_priority = delimiters.get(id(leaf))
2171 if leaf_priority == delimiter_priority:
2174 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2178 and delimiter_priority == COMMA_PRIORITY
2179 and current_line.leaves[-1].type != token.COMMA
2180 and current_line.leaves[-1].type != STANDALONE_COMMENT
2182 current_line.append(Leaf(token.COMMA, ","))
2186 @dont_increase_indentation
2187 def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]:
2188 """Split standalone comments from the rest of the line."""
2189 if not line.contains_standalone_comments(0):
2190 raise CannotSplit("Line does not have any standalone comments")
2192 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2194 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2195 """Append `leaf` to current line or to new line if appending impossible."""
2196 nonlocal current_line
2198 current_line.append_safe(leaf, preformatted=True)
2199 except ValueError as ve:
2202 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2203 current_line.append(leaf)
2205 for index, leaf in enumerate(line.leaves):
2206 yield from append_to_line(leaf)
2208 for comment_after in line.comments_after(leaf, index):
2209 yield from append_to_line(comment_after)
2216 line: Line, py36: bool = False, omit: Collection[LeafID] = ()
2217 ) -> Iterator[Line]:
2218 """Split by rightmost bracket and immediately split contents by a delimiter."""
2219 new_lines = list(right_hand_split(line, py36, omit))
2220 if len(new_lines) != 3:
2221 yield from new_lines
2227 yield from delimiter_split(new_lines[1], py36)
2235 def is_import(leaf: Leaf) -> bool:
2236 """Return True if the given leaf starts an import statement."""
2243 (v == "import" and p and p.type == syms.import_name)
2244 or (v == "from" and p and p.type == syms.import_from)
2249 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
2250 """Leave existing extra newlines if not `inside_brackets`. Remove everything
2253 Note: don't use backslashes for formatting or you'll lose your voting rights.
2255 if not inside_brackets:
2256 spl = leaf.prefix.split("#")
2257 if "\\" not in spl[0]:
2258 nl_count = spl[-1].count("\n")
2261 leaf.prefix = "\n" * nl_count
2267 def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
2268 """Make all string prefixes lowercase.
2270 If remove_u_prefix is given, also removes any u prefix from the string.
2272 Note: Mutates its argument.
2274 match = re.match(r"^([furbFURB]*)(.*)$", leaf.value, re.DOTALL)
2275 assert match is not None, f"failed to match string {leaf.value!r}"
2276 orig_prefix = match.group(1)
2277 new_prefix = orig_prefix.lower()
2279 new_prefix = new_prefix.replace("u", "")
2280 leaf.value = f"{new_prefix}{match.group(2)}"
2283 def normalize_string_quotes(leaf: Leaf) -> None:
2284 """Prefer double quotes but only if it doesn't cause more escaping.
2286 Adds or removes backslashes as appropriate. Doesn't parse and fix
2287 strings nested in f-strings (yet).
2289 Note: Mutates its argument.
2291 value = leaf.value.lstrip("furbFURB")
2292 if value[:3] == '"""':
2295 elif value[:3] == "'''":
2298 elif value[0] == '"':
2304 first_quote_pos = leaf.value.find(orig_quote)
2305 if first_quote_pos == -1:
2306 return # There's an internal error
2308 prefix = leaf.value[:first_quote_pos]
2309 unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
2310 escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}")
2311 escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}")
2312 body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
2313 if "r" in prefix.casefold():
2314 if unescaped_new_quote.search(body):
2315 # There's at least one unescaped new_quote in this raw string
2316 # so converting is impossible
2319 # Do not introduce or remove backslashes in raw strings
2322 # remove unnecessary quotes
2323 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
2324 if body != new_body:
2325 # Consider the string without unnecessary quotes as the original
2327 leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}"
2328 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
2329 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
2330 if new_quote == '"""' and new_body[-1] == '"':
2332 new_body = new_body[:-1] + '\\"'
2333 orig_escape_count = body.count("\\")
2334 new_escape_count = new_body.count("\\")
2335 if new_escape_count > orig_escape_count:
2336 return # Do not introduce more escaping
2338 if new_escape_count == orig_escape_count and orig_quote == '"':
2339 return # Prefer double quotes
2341 leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
2344 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
2345 """Make existing optional parentheses invisible or create new ones.
2347 `parens_after` is a set of string leaf values immeditely after which parens
2350 Standardizes on visible parentheses for single-element tuples, and keeps
2351 existing visible parentheses for other tuples and generator expressions.
2354 for child in list(node.children):
2356 if child.type == syms.atom:
2357 maybe_make_parens_invisible_in_atom(child)
2358 elif is_one_tuple(child):
2359 # wrap child in visible parentheses
2360 lpar = Leaf(token.LPAR, "(")
2361 rpar = Leaf(token.RPAR, ")")
2362 index = child.remove() or 0
2363 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2365 # wrap child in invisible parentheses
2366 lpar = Leaf(token.LPAR, "")
2367 rpar = Leaf(token.RPAR, "")
2368 index = child.remove() or 0
2369 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2371 check_lpar = isinstance(child, Leaf) and child.value in parens_after
2374 def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
2375 """If it's safe, make the parens in the atom `node` invisible, recusively."""
2377 node.type != syms.atom
2378 or is_empty_tuple(node)
2379 or is_one_tuple(node)
2381 or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
2385 first = node.children[0]
2386 last = node.children[-1]
2387 if first.type == token.LPAR and last.type == token.RPAR:
2388 # make parentheses invisible
2389 first.value = "" # type: ignore
2390 last.value = "" # type: ignore
2391 if len(node.children) > 1:
2392 maybe_make_parens_invisible_in_atom(node.children[1])
2398 def is_empty_tuple(node: LN) -> bool:
2399 """Return True if `node` holds an empty tuple."""
2401 node.type == syms.atom
2402 and len(node.children) == 2
2403 and node.children[0].type == token.LPAR
2404 and node.children[1].type == token.RPAR
2408 def is_one_tuple(node: LN) -> bool:
2409 """Return True if `node` holds a tuple with one element, with or without parens."""
2410 if node.type == syms.atom:
2411 if len(node.children) != 3:
2414 lpar, gexp, rpar = node.children
2416 lpar.type == token.LPAR
2417 and gexp.type == syms.testlist_gexp
2418 and rpar.type == token.RPAR
2422 return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
2425 node.type in IMPLICIT_TUPLE
2426 and len(node.children) == 2
2427 and node.children[1].type == token.COMMA
2431 def is_yield(node: LN) -> bool:
2432 """Return True if `node` holds a `yield` or `yield from` expression."""
2433 if node.type == syms.yield_expr:
2436 if node.type == token.NAME and node.value == "yield": # type: ignore
2439 if node.type != syms.atom:
2442 if len(node.children) != 3:
2445 lpar, expr, rpar = node.children
2446 if lpar.type == token.LPAR and rpar.type == token.RPAR:
2447 return is_yield(expr)
2452 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
2453 """Return True if `leaf` is a star or double star in a vararg or kwarg.
2455 If `within` includes VARARGS_PARENTS, this applies to function signatures.
2456 If `within` includes UNPACKING_PARENTS, it applies to right hand-side
2457 extended iterable unpacking (PEP 3132) and additional unpacking
2458 generalizations (PEP 448).
2460 if leaf.type not in STARS or not leaf.parent:
2464 if p.type == syms.star_expr:
2465 # Star expressions are also used as assignment targets in extended
2466 # iterable unpacking (PEP 3132). See what its parent is instead.
2472 return p.type in within
2475 def max_delimiter_priority_in_atom(node: LN) -> int:
2476 """Return maximum delimiter priority inside `node`.
2478 This is specific to atoms with contents contained in a pair of parentheses.
2479 If `node` isn't an atom or there are no enclosing parentheses, returns 0.
2481 if node.type != syms.atom:
2484 first = node.children[0]
2485 last = node.children[-1]
2486 if not (first.type == token.LPAR and last.type == token.RPAR):
2489 bt = BracketTracker()
2490 for c in node.children[1:-1]:
2491 if isinstance(c, Leaf):
2494 for leaf in c.leaves():
2497 return bt.max_delimiter_priority()
2503 def ensure_visible(leaf: Leaf) -> None:
2504 """Make sure parentheses are visible.
2506 They could be invisible as part of some statements (see
2507 :func:`normalize_invible_parens` and :func:`visit_import_from`).
2509 if leaf.type == token.LPAR:
2511 elif leaf.type == token.RPAR:
2515 def is_python36(node: Node) -> bool:
2516 """Return True if the current file is using Python 3.6+ features.
2518 Currently looking for:
2520 - trailing commas after * or ** in function signatures and calls.
2522 for n in node.pre_order():
2523 if n.type == token.STRING:
2524 value_head = n.value[:2] # type: ignore
2525 if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
2529 n.type in {syms.typedargslist, syms.arglist}
2531 and n.children[-1].type == token.COMMA
2533 for ch in n.children:
2534 if ch.type in STARS:
2537 if ch.type == syms.argument:
2538 for argch in ch.children:
2539 if argch.type in STARS:
2545 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
2546 """Generate sets of closing bracket IDs that should be omitted in a RHS.
2548 Brackets can be omitted if the entire trailer up to and including
2549 a preceding closing bracket fits in one line.
2551 Yielded sets are cumulative (contain results of previous yields, too). First
2555 omit: Set[LeafID] = set()
2558 length = 4 * line.depth
2559 opening_bracket = None
2560 closing_bracket = None
2561 optional_brackets: Set[LeafID] = set()
2562 inner_brackets: Set[LeafID] = set()
2563 for index, leaf in enumerate_reversed(line.leaves):
2564 length += len(leaf.prefix) + len(leaf.value)
2565 if length > line_length:
2568 comment: Optional[Leaf]
2569 for comment in line.comments_after(leaf, index):
2570 if "\n" in comment.prefix:
2571 break # Oops, standalone comment!
2573 length += len(comment.value)
2576 if comment is not None:
2577 break # There was a standalone comment, we can't continue.
2579 optional_brackets.discard(id(leaf))
2581 if leaf is opening_bracket:
2582 opening_bracket = None
2583 elif leaf.type in CLOSING_BRACKETS:
2584 inner_brackets.add(id(leaf))
2585 elif leaf.type in CLOSING_BRACKETS:
2587 optional_brackets.add(id(opening_bracket))
2590 if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
2591 # Empty brackets would fail a split so treat them as "inner"
2592 # brackets (e.g. only add them to the `omit` set if another
2593 # pair of brackets was good enough.
2594 inner_brackets.add(id(leaf))
2597 opening_bracket = leaf.opening_bracket
2599 omit.add(id(closing_bracket))
2600 omit.update(inner_brackets)
2601 inner_brackets.clear()
2603 closing_bracket = leaf
2606 def get_future_imports(node: Node) -> Set[str]:
2607 """Return a set of __future__ imports in the file."""
2609 for child in node.children:
2610 if child.type != syms.simple_stmt:
2612 first_child = child.children[0]
2613 if isinstance(first_child, Leaf):
2614 # Continue looking if we see a docstring; otherwise stop.
2616 len(child.children) == 2
2617 and first_child.type == token.STRING
2618 and child.children[1].type == token.NEWLINE
2623 elif first_child.type == syms.import_from:
2624 module_name = first_child.children[1]
2625 if not isinstance(module_name, Leaf) or module_name.value != "__future__":
2627 for import_from_child in first_child.children[3:]:
2628 if isinstance(import_from_child, Leaf):
2629 if import_from_child.type == token.NAME:
2630 imports.add(import_from_child.value)
2632 assert import_from_child.type == syms.import_as_names
2633 for leaf in import_from_child.children:
2634 if isinstance(leaf, Leaf) and leaf.type == token.NAME:
2635 imports.add(leaf.value)
2641 PYTHON_EXTENSIONS = {".py", ".pyi"}
2642 BLACKLISTED_DIRECTORIES = {
2643 "build", "buck-out", "dist", "_build", ".git", ".hg", ".mypy_cache", ".tox", ".venv"
2647 def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
2648 """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
2649 and have one of the PYTHON_EXTENSIONS.
2651 for child in path.iterdir():
2653 if child.name in BLACKLISTED_DIRECTORIES:
2656 yield from gen_python_files_in_dir(child)
2658 elif child.is_file() and child.suffix in PYTHON_EXTENSIONS:
2664 """Provides a reformatting counter. Can be rendered with `str(report)`."""
2667 change_count: int = 0
2669 failure_count: int = 0
2671 def done(self, src: Path, changed: Changed) -> None:
2672 """Increment the counter for successful reformatting. Write out a message."""
2673 if changed is Changed.YES:
2674 reformatted = "would reformat" if self.check else "reformatted"
2676 out(f"{reformatted} {src}")
2677 self.change_count += 1
2680 if changed is Changed.NO:
2681 msg = f"{src} already well formatted, good job."
2683 msg = f"{src} wasn't modified on disk since last run."
2684 out(msg, bold=False)
2685 self.same_count += 1
2687 def failed(self, src: Path, message: str) -> None:
2688 """Increment the counter for failed reformatting. Write out a message."""
2689 err(f"error: cannot format {src}: {message}")
2690 self.failure_count += 1
2693 def return_code(self) -> int:
2694 """Return the exit code that the app should use.
2696 This considers the current state of changed files and failures:
2697 - if there were any failures, return 123;
2698 - if any files were changed and --check is being used, return 1;
2699 - otherwise return 0.
2701 # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
2702 # 126 we have special returncodes reserved by the shell.
2703 if self.failure_count:
2706 elif self.change_count and self.check:
2711 def __str__(self) -> str:
2712 """Render a color report of the current state.
2714 Use `click.unstyle` to remove colors.
2717 reformatted = "would be reformatted"
2718 unchanged = "would be left unchanged"
2719 failed = "would fail to reformat"
2721 reformatted = "reformatted"
2722 unchanged = "left unchanged"
2723 failed = "failed to reformat"
2725 if self.change_count:
2726 s = "s" if self.change_count > 1 else ""
2728 click.style(f"{self.change_count} file{s} {reformatted}", bold=True)
2731 s = "s" if self.same_count > 1 else ""
2732 report.append(f"{self.same_count} file{s} {unchanged}")
2733 if self.failure_count:
2734 s = "s" if self.failure_count > 1 else ""
2736 click.style(f"{self.failure_count} file{s} {failed}", fg="red")
2738 return ", ".join(report) + "."
2741 def assert_equivalent(src: str, dst: str) -> None:
2742 """Raise AssertionError if `src` and `dst` aren't equivalent."""
2747 def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
2748 """Simple visitor generating strings to compare ASTs by content."""
2749 yield f"{' ' * depth}{node.__class__.__name__}("
2751 for field in sorted(node._fields):
2753 value = getattr(node, field)
2754 except AttributeError:
2757 yield f"{' ' * (depth+1)}{field}="
2759 if isinstance(value, list):
2761 if isinstance(item, ast.AST):
2762 yield from _v(item, depth + 2)
2764 elif isinstance(value, ast.AST):
2765 yield from _v(value, depth + 2)
2768 yield f"{' ' * (depth+2)}{value!r}, # {value.__class__.__name__}"
2770 yield f"{' ' * depth}) # /{node.__class__.__name__}"
2773 src_ast = ast.parse(src)
2774 except Exception as exc:
2775 major, minor = sys.version_info[:2]
2776 raise AssertionError(
2777 f"cannot use --safe with this file; failed to parse source file "
2778 f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
2779 f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
2783 dst_ast = ast.parse(dst)
2784 except Exception as exc:
2785 log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
2786 raise AssertionError(
2787 f"INTERNAL ERROR: Black produced invalid code: {exc}. "
2788 f"Please report a bug on https://github.com/ambv/black/issues. "
2789 f"This invalid output might be helpful: {log}"
2792 src_ast_str = "\n".join(_v(src_ast))
2793 dst_ast_str = "\n".join(_v(dst_ast))
2794 if src_ast_str != dst_ast_str:
2795 log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
2796 raise AssertionError(
2797 f"INTERNAL ERROR: Black produced code that is not equivalent to "
2799 f"Please report a bug on https://github.com/ambv/black/issues. "
2800 f"This diff might be helpful: {log}"
2804 def assert_stable(src: str, dst: str, line_length: int, is_pyi: bool = False) -> None:
2805 """Raise AssertionError if `dst` reformats differently the second time."""
2806 newdst = format_str(dst, line_length=line_length, is_pyi=is_pyi)
2809 diff(src, dst, "source", "first pass"),
2810 diff(dst, newdst, "first pass", "second pass"),
2812 raise AssertionError(
2813 f"INTERNAL ERROR: Black produced different code on the second pass "
2814 f"of the formatter. "
2815 f"Please report a bug on https://github.com/ambv/black/issues. "
2816 f"This diff might be helpful: {log}"
2820 def dump_to_file(*output: str) -> str:
2821 """Dump `output` to a temporary file. Return path to the file."""
2824 with tempfile.NamedTemporaryFile(
2825 mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8"
2827 for lines in output:
2829 if lines and lines[-1] != "\n":
2834 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
2835 """Return a unified diff string between strings `a` and `b`."""
2838 a_lines = [line + "\n" for line in a.split("\n")]
2839 b_lines = [line + "\n" for line in b.split("\n")]
2841 difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5)
2845 def cancel(tasks: List[asyncio.Task]) -> None:
2846 """asyncio signal handler that cancels all `tasks` and reports to stderr."""
2852 def shutdown(loop: BaseEventLoop) -> None:
2853 """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
2855 # This part is borrowed from asyncio/runners.py in Python 3.7b2.
2856 to_cancel = [task for task in asyncio.Task.all_tasks(loop) if not task.done()]
2860 for task in to_cancel:
2862 loop.run_until_complete(
2863 asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)
2866 # `concurrent.futures.Future` objects cannot be cancelled once they
2867 # are already running. There might be some when the `shutdown()` happened.
2868 # Silence their logger's spew about the event loop being closed.
2869 cf_logger = logging.getLogger("concurrent.futures")
2870 cf_logger.setLevel(logging.CRITICAL)
2874 def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
2875 """Replace `regex` with `replacement` twice on `original`.
2877 This is used by string normalization to perform replaces on
2878 overlapping matches.
2880 return regex.sub(replacement, regex.sub(replacement, original))
2883 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
2884 """Like `reversed(enumerate(sequence))` if that were possible."""
2885 index = len(sequence) - 1
2886 for element in reversed(sequence):
2887 yield (index, element)
2891 def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
2892 """Return True if `line` is no longer than `line_length`.
2894 Uses the provided `line_str` rendering, if any, otherwise computes a new one.
2897 line_str = str(line).strip("\n")
2899 len(line_str) <= line_length
2900 and "\n" not in line_str # multiline strings
2901 and not line.contains_standalone_comments()
2905 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
2908 def get_cache_file(line_length: int) -> Path:
2909 return CACHE_DIR / f"cache.{line_length}.pickle"
2912 def read_cache(line_length: int) -> Cache:
2913 """Read the cache if it exists and is well formed.
2915 If it is not well formed, the call to write_cache later should resolve the issue.
2917 cache_file = get_cache_file(line_length)
2918 if not cache_file.exists():
2921 with cache_file.open("rb") as fobj:
2923 cache: Cache = pickle.load(fobj)
2924 except pickle.UnpicklingError:
2930 def get_cache_info(path: Path) -> CacheInfo:
2931 """Return the information used to check if a file is already formatted or not."""
2933 return stat.st_mtime, stat.st_size
2937 cache: Cache, sources: Iterable[Path]
2938 ) -> Tuple[List[Path], List[Path]]:
2939 """Split a list of paths into two.
2941 The first list contains paths of files that modified on disk or are not in the
2942 cache. The other list contains paths to non-modified files.
2947 if cache.get(src) != get_cache_info(src):
2954 def write_cache(cache: Cache, sources: List[Path], line_length: int) -> None:
2955 """Update the cache file."""
2956 cache_file = get_cache_file(line_length)
2958 if not CACHE_DIR.exists():
2959 CACHE_DIR.mkdir(parents=True)
2960 new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
2961 with cache_file.open("wb") as fobj:
2962 pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL)
2967 if __name__ == "__main__":