All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 from asyncio.base_events import BaseEventLoop
3 from concurrent.futures import Executor, ProcessPoolExecutor
4 from datetime import datetime
6 from functools import lru_cache, partial, wraps
10 from multiprocessing import Manager, freeze_support
12 from pathlib import Path
39 from appdirs import user_cache_dir
40 from attr import dataclass, evolve, Factory
45 from blib2to3.pytree import Node, Leaf, type_repr
46 from blib2to3 import pygram, pytree
47 from blib2to3.pgen2 import driver, token
48 from blib2to3.pgen2.grammar import Grammar
49 from blib2to3.pgen2.parse import ParseError
52 __version__ = "19.3b0"
53 DEFAULT_LINE_LENGTH = 88
55 r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)/"
57 DEFAULT_INCLUDES = r"\.pyi?$"
58 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
70 LN = Union[Leaf, Node]
71 SplitFunc = Callable[["Line", bool], Iterator["Line"]]
74 CacheInfo = Tuple[Timestamp, FileSize]
75 Cache = Dict[Path, CacheInfo]
76 out = partial(click.secho, bold=True, err=True)
77 err = partial(click.secho, fg="red", err=True)
79 pygram.initialize(CACHE_DIR)
80 syms = pygram.python_symbols
83 class NothingChanged(UserWarning):
84 """Raised when reformatted code is the same as source."""
87 class CannotSplit(Exception):
88 """A readable split that fits the allotted line length is impossible."""
91 class InvalidInput(ValueError):
92 """Raised when input source code fails all parse attempts."""
95 class WriteBack(Enum):
102 def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack":
103 if check and not diff:
106 return cls.DIFF if diff else cls.YES
115 class TargetVersion(Enum):
124 def is_python2(self) -> bool:
125 return self is TargetVersion.PY27
128 PY36_VERSIONS = {TargetVersion.PY36, TargetVersion.PY37, TargetVersion.PY38}
132 # All string literals are unicode
135 NUMERIC_UNDERSCORES = 3
139 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
140 TargetVersion.PY27: set(),
141 TargetVersion.PY33: {Feature.UNICODE_LITERALS},
142 TargetVersion.PY34: {Feature.UNICODE_LITERALS},
143 TargetVersion.PY35: {Feature.UNICODE_LITERALS, Feature.TRAILING_COMMA},
144 TargetVersion.PY36: {
145 Feature.UNICODE_LITERALS,
147 Feature.NUMERIC_UNDERSCORES,
148 Feature.TRAILING_COMMA,
150 TargetVersion.PY37: {
151 Feature.UNICODE_LITERALS,
153 Feature.NUMERIC_UNDERSCORES,
154 Feature.TRAILING_COMMA,
156 TargetVersion.PY38: {
157 Feature.UNICODE_LITERALS,
159 Feature.NUMERIC_UNDERSCORES,
160 Feature.TRAILING_COMMA,
167 target_versions: Set[TargetVersion] = Factory(set)
168 line_length: int = DEFAULT_LINE_LENGTH
169 string_normalization: bool = True
172 def get_cache_key(self) -> str:
173 if self.target_versions:
174 version_str = ",".join(
176 for version in sorted(self.target_versions, key=lambda v: v.value)
182 str(self.line_length),
183 str(int(self.string_normalization)),
184 str(int(self.is_pyi)),
186 return ".".join(parts)
189 def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> bool:
190 return all(feature in VERSION_TO_FEATURES[version] for version in target_versions)
193 def read_pyproject_toml(
194 ctx: click.Context, param: click.Parameter, value: Union[str, int, bool, None]
196 """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
198 Returns the path to a successfully found and read configuration file, None
201 assert not isinstance(value, (int, bool)), "Invalid parameter type passed"
203 root = find_project_root(ctx.params.get("src", ()))
204 path = root / "pyproject.toml"
211 pyproject_toml = toml.load(value)
212 config = pyproject_toml.get("tool", {}).get("black", {})
213 except (toml.TomlDecodeError, OSError) as e:
214 raise click.FileError(
215 filename=value, hint=f"Error reading configuration file: {e}"
221 if ctx.default_map is None:
223 ctx.default_map.update( # type: ignore # bad types in .pyi
224 {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
229 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
234 default=DEFAULT_LINE_LENGTH,
235 help="How many characters per line to allow.",
241 type=click.Choice([v.name.lower() for v in TargetVersion]),
242 callback=lambda c, p, v: [TargetVersion[val.upper()] for val in v],
245 "Python versions that should be supported by Black's output. [default: "
246 "per-file auto-detection]"
253 "Allow using Python 3.6-only syntax on all input files. This will put "
254 "trailing commas in function signatures and calls also after *args and "
255 "**kwargs. Deprecated; use --target-version instead. "
256 "[default: per-file auto-detection]"
263 "Format all input files like typing stubs regardless of file extension "
264 "(useful when piping source on standard input)."
269 "--skip-string-normalization",
271 help="Don't normalize string quotes or prefixes.",
277 "Don't write the files back, just return the status. Return code 0 "
278 "means nothing would change. Return code 1 means some files would be "
279 "reformatted. Return code 123 means there was an internal error."
285 help="Don't write the files back, just output a diff for each file on stdout.",
290 help="If --fast given, skip temporary sanity checks. [default: --safe]",
295 default=DEFAULT_INCLUDES,
297 "A regular expression that matches files and directories that should be "
298 "included on recursive searches. An empty value means all files are "
299 "included regardless of the name. Use forward slashes for directories on "
300 "all platforms (Windows, too). Exclusions are calculated first, inclusions "
308 default=DEFAULT_EXCLUDES,
310 "A regular expression that matches files and directories that should be "
311 "excluded on recursive searches. An empty value means no paths are excluded. "
312 "Use forward slashes for directories on all platforms (Windows, too). "
313 "Exclusions are calculated first, inclusions later."
322 "Don't emit non-error messages to stderr. Errors are still emitted, "
323 "silence those with 2>/dev/null."
331 "Also emit messages to stderr about files that were not changed or were "
332 "ignored due to --exclude=."
335 @click.version_option(version=__version__)
340 exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
347 exists=False, file_okay=True, dir_okay=False, readable=True, allow_dash=False
350 callback=read_pyproject_toml,
351 help="Read configuration from PATH.",
357 target_version: List[TargetVersion],
363 skip_string_normalization: bool,
369 config: Optional[str],
371 """The uncompromising code formatter."""
372 write_back = WriteBack.from_configuration(check=check, diff=diff)
375 err(f"Cannot use both --target-version and --py36")
378 versions = set(target_version)
381 "--py36 is deprecated and will be removed in a future version. "
382 "Use --target-version py36 instead."
384 versions = PY36_VERSIONS
386 # We'll autodetect later.
389 target_versions=versions,
390 line_length=line_length,
392 string_normalization=not skip_string_normalization,
394 if config and verbose:
395 out(f"Using configuration from {config}.", bold=False, fg="blue")
397 include_regex = re_compile_maybe_verbose(include)
399 err(f"Invalid regular expression for include given: {include!r}")
402 exclude_regex = re_compile_maybe_verbose(exclude)
404 err(f"Invalid regular expression for exclude given: {exclude!r}")
406 report = Report(check=check, quiet=quiet, verbose=verbose)
407 root = find_project_root(src)
408 sources: Set[Path] = set()
413 gen_python_files_in_dir(p, root, include_regex, exclude_regex, report)
415 elif p.is_file() or s == "-":
416 # if a file was explicitly given, we don't care about its extension
419 err(f"invalid path: {s}")
420 if len(sources) == 0:
421 if verbose or not quiet:
422 out("No paths given. Nothing to do 😴")
425 if len(sources) == 1:
429 write_back=write_back,
434 loop = asyncio.get_event_loop()
435 executor = ProcessPoolExecutor(max_workers=os.cpu_count())
437 loop.run_until_complete(
441 write_back=write_back,
450 if verbose or not quiet:
451 bang = "💥 💔 💥" if report.return_code else "✨ 🍰 ✨"
452 out(f"All done! {bang}")
453 click.secho(str(report), err=True)
454 ctx.exit(report.return_code)
458 src: Path, fast: bool, write_back: WriteBack, mode: FileMode, report: "Report"
460 """Reformat a single file under `src` without spawning child processes.
462 If `quiet` is True, non-error messages are not output. `line_length`,
463 `write_back`, `fast` and `pyi` options are passed to
464 :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
468 if not src.is_file() and str(src) == "-":
469 if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
470 changed = Changed.YES
473 if write_back != WriteBack.DIFF:
474 cache = read_cache(mode)
475 res_src = src.resolve()
476 if res_src in cache and cache[res_src] == get_cache_info(res_src):
477 changed = Changed.CACHED
478 if changed is not Changed.CACHED and format_file_in_place(
479 src, fast=fast, write_back=write_back, mode=mode
481 changed = Changed.YES
482 if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
483 write_back is WriteBack.CHECK and changed is Changed.NO
485 write_cache(cache, [src], mode)
486 report.done(src, changed)
487 except Exception as exc:
488 report.failed(src, str(exc))
491 async def schedule_formatting(
494 write_back: WriteBack,
500 """Run formatting of `sources` in parallel using the provided `executor`.
502 (Use ProcessPoolExecutors for actual parallelism.)
504 `line_length`, `write_back`, `fast`, and `pyi` options are passed to
505 :func:`format_file_in_place`.
508 if write_back != WriteBack.DIFF:
509 cache = read_cache(mode)
510 sources, cached = filter_cached(cache, sources)
511 for src in sorted(cached):
512 report.done(src, Changed.CACHED)
517 sources_to_cache = []
519 if write_back == WriteBack.DIFF:
520 # For diff output, we need locks to ensure we don't interleave output
521 # from different processes.
523 lock = manager.Lock()
525 loop.run_in_executor(
526 executor, format_file_in_place, src, fast, mode, write_back, lock
528 for src in sorted(sources)
530 pending: Iterable[asyncio.Task] = tasks.keys()
532 loop.add_signal_handler(signal.SIGINT, cancel, pending)
533 loop.add_signal_handler(signal.SIGTERM, cancel, pending)
534 except NotImplementedError:
535 # There are no good alternatives for these on Windows.
538 done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
540 src = tasks.pop(task)
542 cancelled.append(task)
543 elif task.exception():
544 report.failed(src, str(task.exception()))
546 changed = Changed.YES if task.result() else Changed.NO
547 # If the file was written back or was successfully checked as
548 # well-formatted, store this information in the cache.
549 if write_back is WriteBack.YES or (
550 write_back is WriteBack.CHECK and changed is Changed.NO
552 sources_to_cache.append(src)
553 report.done(src, changed)
555 await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
557 write_cache(cache, sources_to_cache, mode)
560 def format_file_in_place(
564 write_back: WriteBack = WriteBack.NO,
565 lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy
567 """Format file under `src` path. Return True if changed.
569 If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
571 `line_length` and `fast` options are passed to :func:`format_file_contents`.
573 if src.suffix == ".pyi":
574 mode = evolve(mode, is_pyi=True)
576 then = datetime.utcfromtimestamp(src.stat().st_mtime)
577 with open(src, "rb") as buf:
578 src_contents, encoding, newline = decode_bytes(buf.read())
580 dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
581 except NothingChanged:
584 if write_back == write_back.YES:
585 with open(src, "w", encoding=encoding, newline=newline) as f:
586 f.write(dst_contents)
587 elif write_back == write_back.DIFF:
588 now = datetime.utcnow()
589 src_name = f"{src}\t{then} +0000"
590 dst_name = f"{src}\t{now} +0000"
591 diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
595 f = io.TextIOWrapper(
601 f.write(diff_contents)
609 def format_stdin_to_stdout(
610 fast: bool, *, write_back: WriteBack = WriteBack.NO, mode: FileMode
612 """Format file on stdin. Return True if changed.
614 If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
615 write a diff to stdout. The `mode` argument is passed to
616 :func:`format_file_contents`.
618 then = datetime.utcnow()
619 src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
622 dst = format_file_contents(src, fast=fast, mode=mode)
625 except NothingChanged:
629 f = io.TextIOWrapper(
630 sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
632 if write_back == WriteBack.YES:
634 elif write_back == WriteBack.DIFF:
635 now = datetime.utcnow()
636 src_name = f"STDIN\t{then} +0000"
637 dst_name = f"STDOUT\t{now} +0000"
638 f.write(diff(src, dst, src_name, dst_name))
642 def format_file_contents(
643 src_contents: str, *, fast: bool, mode: FileMode
645 """Reformat contents a file and return new contents.
647 If `fast` is False, additionally confirm that the reformatted code is
648 valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
649 `line_length` is passed to :func:`format_str`.
651 if src_contents.strip() == "":
654 dst_contents = format_str(src_contents, mode=mode)
655 if src_contents == dst_contents:
659 assert_equivalent(src_contents, dst_contents)
660 assert_stable(src_contents, dst_contents, mode=mode)
664 def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
665 """Reformat a string and return new contents.
667 `line_length` determines how many characters per line are allowed.
669 src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
671 future_imports = get_future_imports(src_node)
672 if mode.target_versions:
673 versions = mode.target_versions
675 versions = detect_target_versions(src_node)
676 normalize_fmt_off(src_node)
677 lines = LineGenerator(
678 remove_u_prefix="unicode_literals" in future_imports
679 or supports_feature(versions, Feature.UNICODE_LITERALS),
681 normalize_strings=mode.string_normalization,
683 elt = EmptyLineTracker(is_pyi=mode.is_pyi)
686 for current_line in lines.visit(src_node):
687 for _ in range(after):
688 dst_contents += str(empty_line)
689 before, after = elt.maybe_empty_lines(current_line)
690 for _ in range(before):
691 dst_contents += str(empty_line)
692 for line in split_line(
694 line_length=mode.line_length,
695 supports_trailing_commas=supports_feature(versions, Feature.TRAILING_COMMA),
697 dst_contents += str(line)
701 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
702 """Return a tuple of (decoded_contents, encoding, newline).
704 `newline` is either CRLF or LF but `decoded_contents` is decoded with
705 universal newlines (i.e. only contains LF).
707 srcbuf = io.BytesIO(src)
708 encoding, lines = tokenize.detect_encoding(srcbuf.readline)
710 return "", encoding, "\n"
712 newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
714 with io.TextIOWrapper(srcbuf, encoding) as tiow:
715 return tiow.read(), encoding, newline
718 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
719 if not target_versions:
720 # No target_version specified, so try all grammars.
722 pygram.python_grammar_no_print_statement_no_exec_statement,
723 pygram.python_grammar_no_print_statement,
724 pygram.python_grammar,
726 elif all(version.is_python2() for version in target_versions):
727 # Python 2-only code, so try Python 2 grammars.
728 return [pygram.python_grammar_no_print_statement, pygram.python_grammar]
730 # Python 3-compatible code, so only try Python 3 grammar.
731 return [pygram.python_grammar_no_print_statement_no_exec_statement]
734 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
735 """Given a string with source, return the lib2to3 Node."""
736 if src_txt[-1:] != "\n":
739 for grammar in get_grammars(set(target_versions)):
740 drv = driver.Driver(grammar, pytree.convert)
742 result = drv.parse_string(src_txt, True)
745 except ParseError as pe:
746 lineno, column = pe.context[1]
747 lines = src_txt.splitlines()
749 faulty_line = lines[lineno - 1]
751 faulty_line = "<line number missing in source>"
752 exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}")
756 if isinstance(result, Leaf):
757 result = Node(syms.file_input, [result])
761 def lib2to3_unparse(node: Node) -> str:
762 """Given a lib2to3 node, return its string representation."""
770 class Visitor(Generic[T]):
771 """Basic lib2to3 visitor that yields things of type `T` on `visit()`."""
773 def visit(self, node: LN) -> Iterator[T]:
774 """Main method to visit `node` and its children.
776 It tries to find a `visit_*()` method for the given `node.type`, like
777 `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.
778 If no dedicated `visit_*()` method is found, chooses `visit_default()`
781 Then yields objects of type `T` from the selected visitor.
784 name = token.tok_name[node.type]
786 name = type_repr(node.type)
787 yield from getattr(self, f"visit_{name}", self.visit_default)(node)
789 def visit_default(self, node: LN) -> Iterator[T]:
790 """Default `visit_*()` implementation. Recurses to children of `node`."""
791 if isinstance(node, Node):
792 for child in node.children:
793 yield from self.visit(child)
797 class DebugVisitor(Visitor[T]):
800 def visit_default(self, node: LN) -> Iterator[T]:
801 indent = " " * (2 * self.tree_depth)
802 if isinstance(node, Node):
803 _type = type_repr(node.type)
804 out(f"{indent}{_type}", fg="yellow")
806 for child in node.children:
807 yield from self.visit(child)
810 out(f"{indent}/{_type}", fg="yellow", bold=False)
812 _type = token.tok_name.get(node.type, str(node.type))
813 out(f"{indent}{_type}", fg="blue", nl=False)
815 # We don't have to handle prefixes for `Node` objects since
816 # that delegates to the first child anyway.
817 out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
818 out(f" {node.value!r}", fg="blue", bold=False)
821 def show(cls, code: Union[str, Leaf, Node]) -> None:
822 """Pretty-print the lib2to3 AST of a given string of `code`.
824 Convenience method for debugging.
826 v: DebugVisitor[None] = DebugVisitor()
827 if isinstance(code, str):
828 code = lib2to3_parse(code)
832 WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
843 STANDALONE_COMMENT = 153
844 token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT"
845 LOGIC_OPERATORS = {"and", "or"}
870 STARS = {token.STAR, token.DOUBLESTAR}
873 syms.argument, # double star in arglist
874 syms.trailer, # single argument to call
876 syms.varargslist, # lambdas
878 UNPACKING_PARENTS = {
879 syms.atom, # single element of a list or set literal
883 syms.testlist_star_expr,
918 COMPREHENSION_PRIORITY = 20
920 TERNARY_PRIORITY = 16
923 COMPARATOR_PRIORITY = 10
934 token.DOUBLESLASH: 4,
944 class BracketTracker:
945 """Keeps track of brackets on a line."""
948 bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
949 delimiters: Dict[LeafID, Priority] = Factory(dict)
950 previous: Optional[Leaf] = None
951 _for_loop_depths: List[int] = Factory(list)
952 _lambda_argument_depths: List[int] = Factory(list)
954 def mark(self, leaf: Leaf) -> None:
955 """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
957 All leaves receive an int `bracket_depth` field that stores how deep
958 within brackets a given leaf is. 0 means there are no enclosing brackets
959 that started on this line.
961 If a leaf is itself a closing bracket, it receives an `opening_bracket`
962 field that it forms a pair with. This is a one-directional link to
963 avoid reference cycles.
965 If a leaf is a delimiter (a token on which Black can split the line if
966 needed) and it's on depth 0, its `id()` is stored in the tracker's
969 if leaf.type == token.COMMENT:
972 self.maybe_decrement_after_for_loop_variable(leaf)
973 self.maybe_decrement_after_lambda_arguments(leaf)
974 if leaf.type in CLOSING_BRACKETS:
976 opening_bracket = self.bracket_match.pop((self.depth, leaf.type))
977 leaf.opening_bracket = opening_bracket
978 leaf.bracket_depth = self.depth
980 delim = is_split_before_delimiter(leaf, self.previous)
981 if delim and self.previous is not None:
982 self.delimiters[id(self.previous)] = delim
984 delim = is_split_after_delimiter(leaf, self.previous)
986 self.delimiters[id(leaf)] = delim
987 if leaf.type in OPENING_BRACKETS:
988 self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
991 self.maybe_increment_lambda_arguments(leaf)
992 self.maybe_increment_for_loop_variable(leaf)
994 def any_open_brackets(self) -> bool:
995 """Return True if there is an yet unmatched open bracket on the line."""
996 return bool(self.bracket_match)
998 def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
999 """Return the highest priority of a delimiter found on the line.
1001 Values are consistent with what `is_split_*_delimiter()` return.
1002 Raises ValueError on no delimiters.
1004 return max(v for k, v in self.delimiters.items() if k not in exclude)
1006 def delimiter_count_with_priority(self, priority: int = 0) -> int:
1007 """Return the number of delimiters with the given `priority`.
1009 If no `priority` is passed, defaults to max priority on the line.
1011 if not self.delimiters:
1014 priority = priority or self.max_delimiter_priority()
1015 return sum(1 for p in self.delimiters.values() if p == priority)
1017 def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
1018 """In a for loop, or comprehension, the variables are often unpacks.
1020 To avoid splitting on the comma in this situation, increase the depth of
1021 tokens between `for` and `in`.
1023 if leaf.type == token.NAME and leaf.value == "for":
1025 self._for_loop_depths.append(self.depth)
1030 def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
1031 """See `maybe_increment_for_loop_variable` above for explanation."""
1033 self._for_loop_depths
1034 and self._for_loop_depths[-1] == self.depth
1035 and leaf.type == token.NAME
1036 and leaf.value == "in"
1039 self._for_loop_depths.pop()
1044 def maybe_increment_lambda_arguments(self, leaf: Leaf) -> bool:
1045 """In a lambda expression, there might be more than one argument.
1047 To avoid splitting on the comma in this situation, increase the depth of
1048 tokens between `lambda` and `:`.
1050 if leaf.type == token.NAME and leaf.value == "lambda":
1052 self._lambda_argument_depths.append(self.depth)
1057 def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
1058 """See `maybe_increment_lambda_arguments` above for explanation."""
1060 self._lambda_argument_depths
1061 and self._lambda_argument_depths[-1] == self.depth
1062 and leaf.type == token.COLON
1065 self._lambda_argument_depths.pop()
1070 def get_open_lsqb(self) -> Optional[Leaf]:
1071 """Return the most recent opening square bracket (if any)."""
1072 return self.bracket_match.get((self.depth - 1, token.RSQB))
1077 """Holds leaves and comments. Can be printed with `str(line)`."""
1080 leaves: List[Leaf] = Factory(list)
1081 comments: Dict[LeafID, List[Leaf]] = Factory(dict) # keys ordered like `leaves`
1082 bracket_tracker: BracketTracker = Factory(BracketTracker)
1083 inside_brackets: bool = False
1084 should_explode: bool = False
1086 def append(self, leaf: Leaf, preformatted: bool = False) -> None:
1087 """Add a new `leaf` to the end of the line.
1089 Unless `preformatted` is True, the `leaf` will receive a new consistent
1090 whitespace prefix and metadata applied by :class:`BracketTracker`.
1091 Trailing commas are maybe removed, unpacked for loop variables are
1092 demoted from being delimiters.
1094 Inline comments are put aside.
1096 has_value = leaf.type in BRACKETS or bool(leaf.value.strip())
1100 if token.COLON == leaf.type and self.is_class_paren_empty:
1101 del self.leaves[-2:]
1102 if self.leaves and not preformatted:
1103 # Note: at this point leaf.prefix should be empty except for
1104 # imports, for which we only preserve newlines.
1105 leaf.prefix += whitespace(
1106 leaf, complex_subscript=self.is_complex_subscript(leaf)
1108 if self.inside_brackets or not preformatted:
1109 self.bracket_tracker.mark(leaf)
1110 self.maybe_remove_trailing_comma(leaf)
1111 if not self.append_comment(leaf):
1112 self.leaves.append(leaf)
1114 def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
1115 """Like :func:`append()` but disallow invalid standalone comment structure.
1117 Raises ValueError when any `leaf` is appended after a standalone comment
1118 or when a standalone comment is not the first leaf on the line.
1120 if self.bracket_tracker.depth == 0:
1122 raise ValueError("cannot append to standalone comments")
1124 if self.leaves and leaf.type == STANDALONE_COMMENT:
1126 "cannot append standalone comments to a populated line"
1129 self.append(leaf, preformatted=preformatted)
1132 def is_comment(self) -> bool:
1133 """Is this line a standalone comment?"""
1134 return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT
1137 def is_decorator(self) -> bool:
1138 """Is this line a decorator?"""
1139 return bool(self) and self.leaves[0].type == token.AT
1142 def is_import(self) -> bool:
1143 """Is this an import line?"""
1144 return bool(self) and is_import(self.leaves[0])
1147 def is_class(self) -> bool:
1148 """Is this line a class definition?"""
1151 and self.leaves[0].type == token.NAME
1152 and self.leaves[0].value == "class"
1156 def is_stub_class(self) -> bool:
1157 """Is this line a class definition with a body consisting only of "..."?"""
1158 return self.is_class and self.leaves[-3:] == [
1159 Leaf(token.DOT, ".") for _ in range(3)
1163 def is_def(self) -> bool:
1164 """Is this a function definition? (Also returns True for async defs.)"""
1166 first_leaf = self.leaves[0]
1171 second_leaf: Optional[Leaf] = self.leaves[1]
1174 return (first_leaf.type == token.NAME and first_leaf.value == "def") or (
1175 first_leaf.type == token.ASYNC
1176 and second_leaf is not None
1177 and second_leaf.type == token.NAME
1178 and second_leaf.value == "def"
1182 def is_class_paren_empty(self) -> bool:
1183 """Is this a class with no base classes but using parentheses?
1185 Those are unnecessary and should be removed.
1189 and len(self.leaves) == 4
1191 and self.leaves[2].type == token.LPAR
1192 and self.leaves[2].value == "("
1193 and self.leaves[3].type == token.RPAR
1194 and self.leaves[3].value == ")"
1198 def is_triple_quoted_string(self) -> bool:
1199 """Is the line a triple quoted string?"""
1202 and self.leaves[0].type == token.STRING
1203 and self.leaves[0].value.startswith(('"""', "'''"))
1206 def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
1207 """If so, needs to be split before emitting."""
1208 for leaf in self.leaves:
1209 if leaf.type == STANDALONE_COMMENT:
1210 if leaf.bracket_depth <= depth_limit:
1214 def contains_inner_type_comments(self) -> bool:
1217 last_leaf = self.leaves[-1]
1218 ignored_ids.add(id(last_leaf))
1219 if last_leaf.type == token.COMMA:
1220 # When trailing commas are inserted by Black for consistency, comments
1221 # after the previous last element are not moved (they don't have to,
1222 # rendering will still be correct). So we ignore trailing commas.
1223 last_leaf = self.leaves[-2]
1224 ignored_ids.add(id(last_leaf))
1228 for leaf_id, comments in self.comments.items():
1229 if leaf_id in ignored_ids:
1232 for comment in comments:
1233 if is_type_comment(comment):
1238 def contains_multiline_strings(self) -> bool:
1239 for leaf in self.leaves:
1240 if is_multiline_string(leaf):
1245 def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
1246 """Remove trailing comma if there is one and it's safe."""
1249 and self.leaves[-1].type == token.COMMA
1250 and closing.type in CLOSING_BRACKETS
1254 if closing.type == token.RBRACE:
1255 self.remove_trailing_comma()
1258 if closing.type == token.RSQB:
1259 comma = self.leaves[-1]
1260 if comma.parent and comma.parent.type == syms.listmaker:
1261 self.remove_trailing_comma()
1264 # For parens let's check if it's safe to remove the comma.
1265 # Imports are always safe.
1267 self.remove_trailing_comma()
1270 # Otherwise, if the trailing one is the only one, we might mistakenly
1271 # change a tuple into a different type by removing the comma.
1272 depth = closing.bracket_depth + 1
1274 opening = closing.opening_bracket
1275 for _opening_index, leaf in enumerate(self.leaves):
1282 for leaf in self.leaves[_opening_index + 1 :]:
1286 bracket_depth = leaf.bracket_depth
1287 if bracket_depth == depth and leaf.type == token.COMMA:
1289 if leaf.parent and leaf.parent.type == syms.arglist:
1294 self.remove_trailing_comma()
1299 def append_comment(self, comment: Leaf) -> bool:
1300 """Add an inline or standalone comment to the line."""
1302 comment.type == STANDALONE_COMMENT
1303 and self.bracket_tracker.any_open_brackets()
1308 if comment.type != token.COMMENT:
1312 comment.type = STANDALONE_COMMENT
1316 self.comments.setdefault(id(self.leaves[-1]), []).append(comment)
1319 def comments_after(self, leaf: Leaf) -> List[Leaf]:
1320 """Generate comments that should appear directly after `leaf`."""
1321 return self.comments.get(id(leaf), [])
1323 def remove_trailing_comma(self) -> None:
1324 """Remove the trailing comma and moves the comments attached to it."""
1325 trailing_comma = self.leaves.pop()
1326 trailing_comma_comments = self.comments.pop(id(trailing_comma), [])
1327 self.comments.setdefault(id(self.leaves[-1]), []).extend(
1328 trailing_comma_comments
1331 def is_complex_subscript(self, leaf: Leaf) -> bool:
1332 """Return True iff `leaf` is part of a slice with non-trivial exprs."""
1333 open_lsqb = self.bracket_tracker.get_open_lsqb()
1334 if open_lsqb is None:
1337 subscript_start = open_lsqb.next_sibling
1339 if isinstance(subscript_start, Node):
1340 if subscript_start.type == syms.listmaker:
1343 if subscript_start.type == syms.subscriptlist:
1344 subscript_start = child_towards(subscript_start, leaf)
1345 return subscript_start is not None and any(
1346 n.type in TEST_DESCENDANTS for n in subscript_start.pre_order()
1349 def __str__(self) -> str:
1350 """Render the line."""
1354 indent = " " * self.depth
1355 leaves = iter(self.leaves)
1356 first = next(leaves)
1357 res = f"{first.prefix}{indent}{first.value}"
1360 for comment in itertools.chain.from_iterable(self.comments.values()):
1364 def __bool__(self) -> bool:
1365 """Return True if the line has leaves or comments."""
1366 return bool(self.leaves or self.comments)
1370 class EmptyLineTracker:
1371 """Provides a stateful method that returns the number of potential extra
1372 empty lines needed before and after the currently processed line.
1374 Note: this tracker works on lines that haven't been split yet. It assumes
1375 the prefix of the first leaf consists of optional newlines. Those newlines
1376 are consumed by `maybe_empty_lines()` and included in the computation.
1379 is_pyi: bool = False
1380 previous_line: Optional[Line] = None
1381 previous_after: int = 0
1382 previous_defs: List[int] = Factory(list)
1384 def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1385 """Return the number of extra empty lines before and after the `current_line`.
1387 This is for separating `def`, `async def` and `class` with extra empty
1388 lines (two on module-level).
1390 before, after = self._maybe_empty_lines(current_line)
1391 before -= self.previous_after
1392 self.previous_after = after
1393 self.previous_line = current_line
1394 return before, after
1396 def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
1398 if current_line.depth == 0:
1399 max_allowed = 1 if self.is_pyi else 2
1400 if current_line.leaves:
1401 # Consume the first leaf's extra newlines.
1402 first_leaf = current_line.leaves[0]
1403 before = first_leaf.prefix.count("\n")
1404 before = min(before, max_allowed)
1405 first_leaf.prefix = ""
1408 depth = current_line.depth
1409 while self.previous_defs and self.previous_defs[-1] >= depth:
1410 self.previous_defs.pop()
1412 before = 0 if depth else 1
1414 before = 1 if depth else 2
1415 if current_line.is_decorator or current_line.is_def or current_line.is_class:
1416 return self._maybe_empty_lines_for_class_or_def(current_line, before)
1420 and self.previous_line.is_import
1421 and not current_line.is_import
1422 and depth == self.previous_line.depth
1424 return (before or 1), 0
1428 and self.previous_line.is_class
1429 and current_line.is_triple_quoted_string
1435 def _maybe_empty_lines_for_class_or_def(
1436 self, current_line: Line, before: int
1437 ) -> Tuple[int, int]:
1438 if not current_line.is_decorator:
1439 self.previous_defs.append(current_line.depth)
1440 if self.previous_line is None:
1441 # Don't insert empty lines before the first line in the file.
1444 if self.previous_line.is_decorator:
1447 if self.previous_line.depth < current_line.depth and (
1448 self.previous_line.is_class or self.previous_line.is_def
1453 self.previous_line.is_comment
1454 and self.previous_line.depth == current_line.depth
1460 if self.previous_line.depth > current_line.depth:
1462 elif current_line.is_class or self.previous_line.is_class:
1463 if current_line.is_stub_class and self.previous_line.is_stub_class:
1464 # No blank line between classes with an empty body
1468 elif current_line.is_def and not self.previous_line.is_def:
1469 # Blank line between a block of functions and a block of non-functions
1475 if current_line.depth and newlines:
1481 class LineGenerator(Visitor[Line]):
1482 """Generates reformatted Line objects. Empty lines are not emitted.
1484 Note: destroys the tree it's visiting by mutating prefixes of its leaves
1485 in ways that will no longer stringify to valid Python code on the tree.
1488 is_pyi: bool = False
1489 normalize_strings: bool = True
1490 current_line: Line = Factory(Line)
1491 remove_u_prefix: bool = False
1493 def line(self, indent: int = 0) -> Iterator[Line]:
1496 If the line is empty, only emit if it makes sense.
1497 If the line is too long, split it first and then generate.
1499 If any lines were generated, set up a new current_line.
1501 if not self.current_line:
1502 self.current_line.depth += indent
1503 return # Line is empty, don't emit. Creating a new one unnecessary.
1505 complete_line = self.current_line
1506 self.current_line = Line(depth=complete_line.depth + indent)
1509 def visit_default(self, node: LN) -> Iterator[Line]:
1510 """Default `visit_*()` implementation. Recurses to children of `node`."""
1511 if isinstance(node, Leaf):
1512 any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
1513 for comment in generate_comments(node):
1514 if any_open_brackets:
1515 # any comment within brackets is subject to splitting
1516 self.current_line.append(comment)
1517 elif comment.type == token.COMMENT:
1518 # regular trailing comment
1519 self.current_line.append(comment)
1520 yield from self.line()
1523 # regular standalone comment
1524 yield from self.line()
1526 self.current_line.append(comment)
1527 yield from self.line()
1529 normalize_prefix(node, inside_brackets=any_open_brackets)
1530 if self.normalize_strings and node.type == token.STRING:
1531 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
1532 normalize_string_quotes(node)
1533 if node.type == token.NUMBER:
1534 normalize_numeric_literal(node)
1535 if node.type not in WHITESPACE:
1536 self.current_line.append(node)
1537 yield from super().visit_default(node)
1539 def visit_INDENT(self, node: Node) -> Iterator[Line]:
1540 """Increase indentation level, maybe yield a line."""
1541 # In blib2to3 INDENT never holds comments.
1542 yield from self.line(+1)
1543 yield from self.visit_default(node)
1545 def visit_DEDENT(self, node: Node) -> Iterator[Line]:
1546 """Decrease indentation level, maybe yield a line."""
1547 # The current line might still wait for trailing comments. At DEDENT time
1548 # there won't be any (they would be prefixes on the preceding NEWLINE).
1549 # Emit the line then.
1550 yield from self.line()
1552 # While DEDENT has no value, its prefix may contain standalone comments
1553 # that belong to the current indentation level. Get 'em.
1554 yield from self.visit_default(node)
1556 # Finally, emit the dedent.
1557 yield from self.line(-1)
1560 self, node: Node, keywords: Set[str], parens: Set[str]
1561 ) -> Iterator[Line]:
1562 """Visit a statement.
1564 This implementation is shared for `if`, `while`, `for`, `try`, `except`,
1565 `def`, `with`, `class`, `assert` and assignments.
1567 The relevant Python language `keywords` for a given statement will be
1568 NAME leaves within it. This methods puts those on a separate line.
1570 `parens` holds a set of string leaf values immediately after which
1571 invisible parens should be put.
1573 normalize_invisible_parens(node, parens_after=parens)
1574 for child in node.children:
1575 if child.type == token.NAME and child.value in keywords: # type: ignore
1576 yield from self.line()
1578 yield from self.visit(child)
1580 def visit_suite(self, node: Node) -> Iterator[Line]:
1581 """Visit a suite."""
1582 if self.is_pyi and is_stub_suite(node):
1583 yield from self.visit(node.children[2])
1585 yield from self.visit_default(node)
1587 def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
1588 """Visit a statement without nested statements."""
1589 is_suite_like = node.parent and node.parent.type in STATEMENT
1591 if self.is_pyi and is_stub_body(node):
1592 yield from self.visit_default(node)
1594 yield from self.line(+1)
1595 yield from self.visit_default(node)
1596 yield from self.line(-1)
1599 if not self.is_pyi or not node.parent or not is_stub_suite(node.parent):
1600 yield from self.line()
1601 yield from self.visit_default(node)
1603 def visit_async_stmt(self, node: Node) -> Iterator[Line]:
1604 """Visit `async def`, `async for`, `async with`."""
1605 yield from self.line()
1607 children = iter(node.children)
1608 for child in children:
1609 yield from self.visit(child)
1611 if child.type == token.ASYNC:
1614 internal_stmt = next(children)
1615 for child in internal_stmt.children:
1616 yield from self.visit(child)
1618 def visit_decorators(self, node: Node) -> Iterator[Line]:
1619 """Visit decorators."""
1620 for child in node.children:
1621 yield from self.line()
1622 yield from self.visit(child)
1624 def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
1625 """Remove a semicolon and put the other statement on a separate line."""
1626 yield from self.line()
1628 def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
1629 """End of file. Process outstanding comments and end with a newline."""
1630 yield from self.visit_default(leaf)
1631 yield from self.line()
1633 def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
1634 if not self.current_line.bracket_tracker.any_open_brackets():
1635 yield from self.line()
1636 yield from self.visit_default(leaf)
1638 def __attrs_post_init__(self) -> None:
1639 """You are in a twisty little maze of passages."""
1642 self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
1643 self.visit_if_stmt = partial(
1644 v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
1646 self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
1647 self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
1648 self.visit_try_stmt = partial(
1649 v, keywords={"try", "except", "else", "finally"}, parens=Ø
1651 self.visit_except_clause = partial(v, keywords={"except"}, parens=Ø)
1652 self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
1653 self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
1654 self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
1655 self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
1656 self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
1657 self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
1658 self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"})
1659 self.visit_async_funcdef = self.visit_async_stmt
1660 self.visit_decorated = self.visit_decorators
1663 IMPLICIT_TUPLE = {syms.testlist, syms.testlist_star_expr, syms.exprlist}
1664 BRACKET = {token.LPAR: token.RPAR, token.LSQB: token.RSQB, token.LBRACE: token.RBRACE}
1665 OPENING_BRACKETS = set(BRACKET.keys())
1666 CLOSING_BRACKETS = set(BRACKET.values())
1667 BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS
1668 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
1671 def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901
1672 """Return whitespace prefix if needed for the given `leaf`.
1674 `complex_subscript` signals whether the given leaf is part of a subscription
1675 which has non-trivial arguments, like arithmetic expressions or function calls.
1683 if t in ALWAYS_NO_SPACE:
1686 if t == token.COMMENT:
1689 assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
1690 if t == token.COLON and p.type not in {
1697 prev = leaf.prev_sibling
1699 prevp = preceding_leaf(p)
1700 if not prevp or prevp.type in OPENING_BRACKETS:
1703 if t == token.COLON:
1704 if prevp.type == token.COLON:
1707 elif prevp.type != token.COMMA and not complex_subscript:
1712 if prevp.type == token.EQUAL:
1714 if prevp.parent.type in {
1722 elif prevp.parent.type == syms.typedargslist:
1723 # A bit hacky: if the equal sign has whitespace, it means we
1724 # previously found it's a typed argument. So, we're using
1728 elif prevp.type in STARS:
1729 if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1732 elif prevp.type == token.COLON:
1733 if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
1734 return SPACE if complex_subscript else NO
1738 and prevp.parent.type == syms.factor
1739 and prevp.type in MATH_OPERATORS
1744 prevp.type == token.RIGHTSHIFT
1746 and prevp.parent.type == syms.shift_expr
1747 and prevp.prev_sibling
1748 and prevp.prev_sibling.type == token.NAME
1749 and prevp.prev_sibling.value == "print" # type: ignore
1751 # Python 2 print chevron
1754 elif prev.type in OPENING_BRACKETS:
1757 if p.type in {syms.parameters, syms.arglist}:
1758 # untyped function signatures or calls
1759 if not prev or prev.type != token.COMMA:
1762 elif p.type == syms.varargslist:
1764 if prev and prev.type != token.COMMA:
1767 elif p.type == syms.typedargslist:
1768 # typed function signatures
1772 if t == token.EQUAL:
1773 if prev.type != syms.tname:
1776 elif prev.type == token.EQUAL:
1777 # A bit hacky: if the equal sign has whitespace, it means we
1778 # previously found it's a typed argument. So, we're using that, too.
1781 elif prev.type != token.COMMA:
1784 elif p.type == syms.tname:
1787 prevp = preceding_leaf(p)
1788 if not prevp or prevp.type != token.COMMA:
1791 elif p.type == syms.trailer:
1792 # attributes and calls
1793 if t == token.LPAR or t == token.RPAR:
1798 prevp = preceding_leaf(p)
1799 if not prevp or prevp.type != token.NUMBER:
1802 elif t == token.LSQB:
1805 elif prev.type != token.COMMA:
1808 elif p.type == syms.argument:
1810 if t == token.EQUAL:
1814 prevp = preceding_leaf(p)
1815 if not prevp or prevp.type == token.LPAR:
1818 elif prev.type in {token.EQUAL} | STARS:
1821 elif p.type == syms.decorator:
1825 elif p.type == syms.dotted_name:
1829 prevp = preceding_leaf(p)
1830 if not prevp or prevp.type == token.AT or prevp.type == token.DOT:
1833 elif p.type == syms.classdef:
1837 if prev and prev.type == token.LPAR:
1840 elif p.type in {syms.subscript, syms.sliceop}:
1843 assert p.parent is not None, "subscripts are always parented"
1844 if p.parent.type == syms.subscriptlist:
1849 elif not complex_subscript:
1852 elif p.type == syms.atom:
1853 if prev and t == token.DOT:
1854 # dots, but not the first one.
1857 elif p.type == syms.dictsetmaker:
1859 if prev and prev.type == token.DOUBLESTAR:
1862 elif p.type in {syms.factor, syms.star_expr}:
1865 prevp = preceding_leaf(p)
1866 if not prevp or prevp.type in OPENING_BRACKETS:
1869 prevp_parent = prevp.parent
1870 assert prevp_parent is not None
1871 if prevp.type == token.COLON and prevp_parent.type in {
1877 elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:
1880 elif t in {token.NAME, token.NUMBER, token.STRING}:
1883 elif p.type == syms.import_from:
1885 if prev and prev.type == token.DOT:
1888 elif t == token.NAME:
1892 if prev and prev.type == token.DOT:
1895 elif p.type == syms.sliceop:
1901 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
1902 """Return the first leaf that precedes `node`, if any."""
1904 res = node.prev_sibling
1906 if isinstance(res, Leaf):
1910 return list(res.leaves())[-1]
1919 def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
1920 """Return the child of `ancestor` that contains `descendant`."""
1921 node: Optional[LN] = descendant
1922 while node and node.parent != ancestor:
1927 def container_of(leaf: Leaf) -> LN:
1928 """Return `leaf` or one of its ancestors that is the topmost container of it.
1930 By "container" we mean a node where `leaf` is the very first child.
1932 same_prefix = leaf.prefix
1933 container: LN = leaf
1935 parent = container.parent
1939 if parent.children[0].prefix != same_prefix:
1942 if parent.type == syms.file_input:
1945 if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:
1952 def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:
1953 """Return the priority of the `leaf` delimiter, given a line break after it.
1955 The delimiter priorities returned here are from those delimiters that would
1956 cause a line break after themselves.
1958 Higher numbers are higher priority.
1960 if leaf.type == token.COMMA:
1961 return COMMA_PRIORITY
1966 def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:
1967 """Return the priority of the `leaf` delimiter, given a line break before it.
1969 The delimiter priorities returned here are from those delimiters that would
1970 cause a line break before themselves.
1972 Higher numbers are higher priority.
1974 if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS):
1975 # * and ** might also be MATH_OPERATORS but in this case they are not.
1976 # Don't treat them as a delimiter.
1980 leaf.type == token.DOT
1982 and leaf.parent.type not in {syms.import_from, syms.dotted_name}
1983 and (previous is None or previous.type in CLOSING_BRACKETS)
1988 leaf.type in MATH_OPERATORS
1990 and leaf.parent.type not in {syms.factor, syms.star_expr}
1992 return MATH_PRIORITIES[leaf.type]
1994 if leaf.type in COMPARATORS:
1995 return COMPARATOR_PRIORITY
1998 leaf.type == token.STRING
1999 and previous is not None
2000 and previous.type == token.STRING
2002 return STRING_PRIORITY
2004 if leaf.type not in {token.NAME, token.ASYNC}:
2010 and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
2011 or leaf.type == token.ASYNC
2014 not isinstance(leaf.prev_sibling, Leaf)
2015 or leaf.prev_sibling.value != "async"
2017 return COMPREHENSION_PRIORITY
2022 and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
2024 return COMPREHENSION_PRIORITY
2026 if leaf.value in {"if", "else"} and leaf.parent and leaf.parent.type == syms.test:
2027 return TERNARY_PRIORITY
2029 if leaf.value == "is":
2030 return COMPARATOR_PRIORITY
2035 and leaf.parent.type in {syms.comp_op, syms.comparison}
2037 previous is not None
2038 and previous.type == token.NAME
2039 and previous.value == "not"
2042 return COMPARATOR_PRIORITY
2047 and leaf.parent.type == syms.comp_op
2049 previous is not None
2050 and previous.type == token.NAME
2051 and previous.value == "is"
2054 return COMPARATOR_PRIORITY
2056 if leaf.value in LOGIC_OPERATORS and leaf.parent:
2057 return LOGIC_PRIORITY
2062 FMT_OFF = {"# fmt: off", "# fmt:off", "# yapf: disable"}
2063 FMT_ON = {"# fmt: on", "# fmt:on", "# yapf: enable"}
2066 def generate_comments(leaf: LN) -> Iterator[Leaf]:
2067 """Clean the prefix of the `leaf` and generate comments from it, if any.
2069 Comments in lib2to3 are shoved into the whitespace prefix. This happens
2070 in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
2071 move because it does away with modifying the grammar to include all the
2072 possible places in which comments can be placed.
2074 The sad consequence for us though is that comments don't "belong" anywhere.
2075 This is why this function generates simple parentless Leaf objects for
2076 comments. We simply don't know what the correct parent should be.
2078 No matter though, we can live without this. We really only need to
2079 differentiate between inline and standalone comments. The latter don't
2080 share the line with any code.
2082 Inline comments are emitted as regular token.COMMENT leaves. Standalone
2083 are emitted with a fake STANDALONE_COMMENT token identifier.
2085 for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):
2086 yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines)
2091 """Describes a piece of syntax that is a comment.
2093 It's not a :class:`blib2to3.pytree.Leaf` so that:
2095 * it can be cached (`Leaf` objects should not be reused more than once as
2096 they store their lineno, column, prefix, and parent information);
2097 * `newlines` and `consumed` fields are kept separate from the `value`. This
2098 simplifies handling of special marker comments like ``# fmt: off/on``.
2101 type: int # token.COMMENT or STANDALONE_COMMENT
2102 value: str # content of the comment
2103 newlines: int # how many newlines before the comment
2104 consumed: int # how many characters of the original leaf's prefix did we consume
2107 @lru_cache(maxsize=4096)
2108 def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
2109 """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
2110 result: List[ProtoComment] = []
2111 if not prefix or "#" not in prefix:
2116 for index, line in enumerate(prefix.split("\n")):
2117 consumed += len(line) + 1 # adding the length of the split '\n'
2118 line = line.lstrip()
2121 if not line.startswith("#"):
2124 if index == 0 and not is_endmarker:
2125 comment_type = token.COMMENT # simple trailing comment
2127 comment_type = STANDALONE_COMMENT
2128 comment = make_comment(line)
2131 type=comment_type, value=comment, newlines=nlines, consumed=consumed
2138 def make_comment(content: str) -> str:
2139 """Return a consistently formatted comment from the given `content` string.
2141 All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single
2142 space between the hash sign and the content.
2144 If `content` didn't start with a hash sign, one is provided.
2146 content = content.rstrip()
2150 if content[0] == "#":
2151 content = content[1:]
2152 if content and content[0] not in " !:#'%":
2153 content = " " + content
2154 return "#" + content
2160 inner: bool = False,
2161 supports_trailing_commas: bool = False,
2162 ) -> Iterator[Line]:
2163 """Split a `line` into potentially many lines.
2165 They should fit in the allotted `line_length` but might not be able to.
2166 `inner` signifies that there were a pair of brackets somewhere around the
2167 current `line`, possibly transitively. This means we can fallback to splitting
2168 by delimiters if the LHS/RHS don't yield any results.
2170 If `supports_trailing_commas` is True, splitting may use the TRAILING_COMMA feature.
2176 line_str = str(line).strip("\n")
2179 not line.contains_inner_type_comments()
2180 and not line.should_explode
2181 and is_line_short_enough(line, line_length=line_length, line_str=line_str)
2186 split_funcs: List[SplitFunc]
2188 split_funcs = [left_hand_split]
2191 def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
2192 for omit in generate_trailers_to_omit(line, line_length):
2195 line, line_length, supports_trailing_commas, omit=omit
2198 if is_line_short_enough(lines[0], line_length=line_length):
2202 # All splits failed, best effort split with no omits.
2203 # This mostly happens to multiline strings that are by definition
2204 # reported as not fitting a single line.
2205 yield from right_hand_split(line, line_length, supports_trailing_commas)
2207 if line.inside_brackets:
2208 split_funcs = [delimiter_split, standalone_comment_split, rhs]
2211 for split_func in split_funcs:
2212 # We are accumulating lines in `result` because we might want to abort
2213 # mission and return the original line in the end, or attempt a different
2215 result: List[Line] = []
2217 for l in split_func(line, supports_trailing_commas):
2218 if str(l).strip("\n") == line_str:
2219 raise CannotSplit("Split function returned an unchanged result")
2224 line_length=line_length,
2226 supports_trailing_commas=supports_trailing_commas,
2240 def left_hand_split(
2241 line: Line, supports_trailing_commas: bool = False
2242 ) -> Iterator[Line]:
2243 """Split line into many lines, starting with the first matching bracket pair.
2245 Note: this usually looks weird, only use this for function definitions.
2246 Prefer RHS otherwise. This is why this function is not symmetrical with
2247 :func:`right_hand_split` which also handles optional parentheses.
2249 tail_leaves: List[Leaf] = []
2250 body_leaves: List[Leaf] = []
2251 head_leaves: List[Leaf] = []
2252 current_leaves = head_leaves
2253 matching_bracket = None
2254 for leaf in line.leaves:
2256 current_leaves is body_leaves
2257 and leaf.type in CLOSING_BRACKETS
2258 and leaf.opening_bracket is matching_bracket
2260 current_leaves = tail_leaves if body_leaves else head_leaves
2261 current_leaves.append(leaf)
2262 if current_leaves is head_leaves:
2263 if leaf.type in OPENING_BRACKETS:
2264 matching_bracket = leaf
2265 current_leaves = body_leaves
2266 if not matching_bracket:
2267 raise CannotSplit("No brackets found")
2269 head = bracket_split_build_line(head_leaves, line, matching_bracket)
2270 body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
2271 tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
2272 bracket_split_succeeded_or_raise(head, body, tail)
2273 for result in (head, body, tail):
2278 def right_hand_split(
2281 supports_trailing_commas: bool = False,
2282 omit: Collection[LeafID] = (),
2283 ) -> Iterator[Line]:
2284 """Split line into many lines, starting with the last matching bracket pair.
2286 If the split was by optional parentheses, attempt splitting without them, too.
2287 `omit` is a collection of closing bracket IDs that shouldn't be considered for
2290 Note: running this function modifies `bracket_depth` on the leaves of `line`.
2292 tail_leaves: List[Leaf] = []
2293 body_leaves: List[Leaf] = []
2294 head_leaves: List[Leaf] = []
2295 current_leaves = tail_leaves
2296 opening_bracket = None
2297 closing_bracket = None
2298 for leaf in reversed(line.leaves):
2299 if current_leaves is body_leaves:
2300 if leaf is opening_bracket:
2301 current_leaves = head_leaves if body_leaves else tail_leaves
2302 current_leaves.append(leaf)
2303 if current_leaves is tail_leaves:
2304 if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
2305 opening_bracket = leaf.opening_bracket
2306 closing_bracket = leaf
2307 current_leaves = body_leaves
2308 if not (opening_bracket and closing_bracket and head_leaves):
2309 # If there is no opening or closing_bracket that means the split failed and
2310 # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
2311 # the matching `opening_bracket` wasn't available on `line` anymore.
2312 raise CannotSplit("No brackets found")
2314 tail_leaves.reverse()
2315 body_leaves.reverse()
2316 head_leaves.reverse()
2317 head = bracket_split_build_line(head_leaves, line, opening_bracket)
2318 body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
2319 tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
2320 bracket_split_succeeded_or_raise(head, body, tail)
2322 # the body shouldn't be exploded
2323 not body.should_explode
2324 # the opening bracket is an optional paren
2325 and opening_bracket.type == token.LPAR
2326 and not opening_bracket.value
2327 # the closing bracket is an optional paren
2328 and closing_bracket.type == token.RPAR
2329 and not closing_bracket.value
2330 # it's not an import (optional parens are the only thing we can split on
2331 # in this case; attempting a split without them is a waste of time)
2332 and not line.is_import
2333 # there are no standalone comments in the body
2334 and not body.contains_standalone_comments(0)
2335 # and we can actually remove the parens
2336 and can_omit_invisible_parens(body, line_length)
2338 omit = {id(closing_bracket), *omit}
2340 yield from right_hand_split(
2343 supports_trailing_commas=supports_trailing_commas,
2351 or is_line_short_enough(body, line_length=line_length)
2354 "Splitting failed, body is still too long and can't be split."
2357 elif head.contains_multiline_strings() or tail.contains_multiline_strings():
2359 "The current optional pair of parentheses is bound to fail to "
2360 "satisfy the splitting algorithm because the head or the tail "
2361 "contains multiline strings which by definition never fit one "
2365 ensure_visible(opening_bracket)
2366 ensure_visible(closing_bracket)
2367 for result in (head, body, tail):
2372 def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
2373 """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
2375 Do nothing otherwise.
2377 A left- or right-hand split is based on a pair of brackets. Content before
2378 (and including) the opening bracket is left on one line, content inside the
2379 brackets is put on a separate line, and finally content starting with and
2380 following the closing bracket is put on a separate line.
2382 Those are called `head`, `body`, and `tail`, respectively. If the split
2383 produced the same line (all content in `head`) or ended up with an empty `body`
2384 and the `tail` is just the closing bracket, then it's considered failed.
2386 tail_len = len(str(tail).strip())
2389 raise CannotSplit("Splitting brackets produced the same line")
2393 f"Splitting brackets on an empty body to save "
2394 f"{tail_len} characters is not worth it"
2398 def bracket_split_build_line(
2399 leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
2401 """Return a new line with given `leaves` and respective comments from `original`.
2403 If `is_body` is True, the result line is one-indented inside brackets and as such
2404 has its first leaf's prefix normalized and a trailing comma added when expected.
2406 result = Line(depth=original.depth)
2408 result.inside_brackets = True
2411 # Since body is a new indent level, remove spurious leading whitespace.
2412 normalize_prefix(leaves[0], inside_brackets=True)
2413 # Ensure a trailing comma when expected.
2414 if original.is_import:
2415 if leaves[-1].type != token.COMMA:
2416 leaves.append(Leaf(token.COMMA, ","))
2419 result.append(leaf, preformatted=True)
2420 for comment_after in original.comments_after(leaf):
2421 result.append(comment_after, preformatted=True)
2423 result.should_explode = should_explode(result, opening_bracket)
2427 def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
2428 """Normalize prefix of the first leaf in every line returned by `split_func`.
2430 This is a decorator over relevant split functions.
2435 line: Line, supports_trailing_commas: bool = False
2436 ) -> Iterator[Line]:
2437 for l in split_func(line, supports_trailing_commas):
2438 normalize_prefix(l.leaves[0], inside_brackets=True)
2441 return split_wrapper
2444 @dont_increase_indentation
2445 def delimiter_split(
2446 line: Line, supports_trailing_commas: bool = False
2447 ) -> Iterator[Line]:
2448 """Split according to delimiters of the highest priority.
2450 If `supports_trailing_commas` is True, the split will add trailing commas
2451 also in function signatures that contain `*` and `**`.
2454 last_leaf = line.leaves[-1]
2456 raise CannotSplit("Line empty")
2458 bt = line.bracket_tracker
2460 delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
2462 raise CannotSplit("No delimiters found")
2464 if delimiter_priority == DOT_PRIORITY:
2465 if bt.delimiter_count_with_priority(delimiter_priority) == 1:
2466 raise CannotSplit("Splitting a single attribute from its owner looks wrong")
2468 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2469 lowest_depth = sys.maxsize
2470 trailing_comma_safe = True
2472 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2473 """Append `leaf` to current line or to new line if appending impossible."""
2474 nonlocal current_line
2476 current_line.append_safe(leaf, preformatted=True)
2480 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2481 current_line.append(leaf)
2483 for leaf in line.leaves:
2484 yield from append_to_line(leaf)
2486 for comment_after in line.comments_after(leaf):
2487 yield from append_to_line(comment_after)
2489 lowest_depth = min(lowest_depth, leaf.bracket_depth)
2490 if leaf.bracket_depth == lowest_depth and is_vararg(
2491 leaf, within=VARARGS_PARENTS
2493 trailing_comma_safe = trailing_comma_safe and supports_trailing_commas
2494 leaf_priority = bt.delimiters.get(id(leaf))
2495 if leaf_priority == delimiter_priority:
2498 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2502 and delimiter_priority == COMMA_PRIORITY
2503 and current_line.leaves[-1].type != token.COMMA
2504 and current_line.leaves[-1].type != STANDALONE_COMMENT
2506 current_line.append(Leaf(token.COMMA, ","))
2510 @dont_increase_indentation
2511 def standalone_comment_split(
2512 line: Line, supports_trailing_commas: bool = False
2513 ) -> Iterator[Line]:
2514 """Split standalone comments from the rest of the line."""
2515 if not line.contains_standalone_comments(0):
2516 raise CannotSplit("Line does not have any standalone comments")
2518 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2520 def append_to_line(leaf: Leaf) -> Iterator[Line]:
2521 """Append `leaf` to current line or to new line if appending impossible."""
2522 nonlocal current_line
2524 current_line.append_safe(leaf, preformatted=True)
2528 current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
2529 current_line.append(leaf)
2531 for leaf in line.leaves:
2532 yield from append_to_line(leaf)
2534 for comment_after in line.comments_after(leaf):
2535 yield from append_to_line(comment_after)
2541 def is_import(leaf: Leaf) -> bool:
2542 """Return True if the given leaf starts an import statement."""
2549 (v == "import" and p and p.type == syms.import_name)
2550 or (v == "from" and p and p.type == syms.import_from)
2555 def is_type_comment(leaf: Leaf) -> bool:
2556 """Return True if the given leaf is a special comment.
2557 Only returns true for type comments for now."""
2560 return t in {token.COMMENT, t == STANDALONE_COMMENT} and v.startswith("# type:")
2563 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
2564 """Leave existing extra newlines if not `inside_brackets`. Remove everything
2567 Note: don't use backslashes for formatting or you'll lose your voting rights.
2569 if not inside_brackets:
2570 spl = leaf.prefix.split("#")
2571 if "\\" not in spl[0]:
2572 nl_count = spl[-1].count("\n")
2575 leaf.prefix = "\n" * nl_count
2581 def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
2582 """Make all string prefixes lowercase.
2584 If remove_u_prefix is given, also removes any u prefix from the string.
2586 Note: Mutates its argument.
2588 match = re.match(r"^([furbFURB]*)(.*)$", leaf.value, re.DOTALL)
2589 assert match is not None, f"failed to match string {leaf.value!r}"
2590 orig_prefix = match.group(1)
2591 new_prefix = orig_prefix.lower()
2593 new_prefix = new_prefix.replace("u", "")
2594 leaf.value = f"{new_prefix}{match.group(2)}"
2597 def normalize_string_quotes(leaf: Leaf) -> None:
2598 """Prefer double quotes but only if it doesn't cause more escaping.
2600 Adds or removes backslashes as appropriate. Doesn't parse and fix
2601 strings nested in f-strings (yet).
2603 Note: Mutates its argument.
2605 value = leaf.value.lstrip("furbFURB")
2606 if value[:3] == '"""':
2609 elif value[:3] == "'''":
2612 elif value[0] == '"':
2618 first_quote_pos = leaf.value.find(orig_quote)
2619 if first_quote_pos == -1:
2620 return # There's an internal error
2622 prefix = leaf.value[:first_quote_pos]
2623 unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
2624 escaped_new_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
2625 escaped_orig_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
2626 body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
2627 if "r" in prefix.casefold():
2628 if unescaped_new_quote.search(body):
2629 # There's at least one unescaped new_quote in this raw string
2630 # so converting is impossible
2633 # Do not introduce or remove backslashes in raw strings
2636 # remove unnecessary escapes
2637 new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
2638 if body != new_body:
2639 # Consider the string without unnecessary escapes as the original
2641 leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}"
2642 new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
2643 new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
2644 if "f" in prefix.casefold():
2645 matches = re.findall(r"[^{]\{(.*?)\}[^}]", new_body)
2648 # Do not introduce backslashes in interpolated expressions
2650 if new_quote == '"""' and new_body[-1:] == '"':
2652 new_body = new_body[:-1] + '\\"'
2653 orig_escape_count = body.count("\\")
2654 new_escape_count = new_body.count("\\")
2655 if new_escape_count > orig_escape_count:
2656 return # Do not introduce more escaping
2658 if new_escape_count == orig_escape_count and orig_quote == '"':
2659 return # Prefer double quotes
2661 leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
2664 def normalize_numeric_literal(leaf: Leaf) -> None:
2665 """Normalizes numeric (float, int, and complex) literals.
2667 All letters used in the representation are normalized to lowercase (except
2668 in Python 2 long literals).
2670 text = leaf.value.lower()
2671 if text.startswith(("0o", "0b")):
2672 # Leave octal and binary literals alone.
2674 elif text.startswith("0x"):
2675 # Change hex literals to upper case.
2676 before, after = text[:2], text[2:]
2677 text = f"{before}{after.upper()}"
2679 before, after = text.split("e")
2681 if after.startswith("-"):
2684 elif after.startswith("+"):
2686 before = format_float_or_int_string(before)
2687 text = f"{before}e{sign}{after}"
2688 elif text.endswith(("j", "l")):
2691 # Capitalize in "2L" because "l" looks too similar to "1".
2694 text = f"{format_float_or_int_string(number)}{suffix}"
2696 text = format_float_or_int_string(text)
2700 def format_float_or_int_string(text: str) -> str:
2701 """Formats a float string like "1.0"."""
2705 before, after = text.split(".")
2706 return f"{before or 0}.{after or 0}"
2709 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
2710 """Make existing optional parentheses invisible or create new ones.
2712 `parens_after` is a set of string leaf values immeditely after which parens
2715 Standardizes on visible parentheses for single-element tuples, and keeps
2716 existing visible parentheses for other tuples and generator expressions.
2718 for pc in list_comments(node.prefix, is_endmarker=False):
2719 if pc.value in FMT_OFF:
2720 # This `node` has a prefix with `# fmt: off`, don't mess with parens.
2724 for index, child in enumerate(list(node.children)):
2726 if child.type == syms.atom:
2727 if maybe_make_parens_invisible_in_atom(child):
2728 lpar = Leaf(token.LPAR, "")
2729 rpar = Leaf(token.RPAR, "")
2730 index = child.remove() or 0
2731 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2732 elif is_one_tuple(child):
2733 # wrap child in visible parentheses
2734 lpar = Leaf(token.LPAR, "(")
2735 rpar = Leaf(token.RPAR, ")")
2737 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2738 elif node.type == syms.import_from:
2739 # "import from" nodes store parentheses directly as part of
2741 if child.type == token.LPAR:
2742 # make parentheses invisible
2743 child.value = "" # type: ignore
2744 node.children[-1].value = "" # type: ignore
2745 elif child.type != token.STAR:
2746 # insert invisible parentheses
2747 node.insert_child(index, Leaf(token.LPAR, ""))
2748 node.append_child(Leaf(token.RPAR, ""))
2751 elif not (isinstance(child, Leaf) and is_multiline_string(child)):
2752 # wrap child in invisible parentheses
2753 lpar = Leaf(token.LPAR, "")
2754 rpar = Leaf(token.RPAR, "")
2755 index = child.remove() or 0
2756 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
2758 check_lpar = isinstance(child, Leaf) and child.value in parens_after
2761 def normalize_fmt_off(node: Node) -> None:
2762 """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
2765 try_again = convert_one_fmt_off_pair(node)
2768 def convert_one_fmt_off_pair(node: Node) -> bool:
2769 """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
2771 Returns True if a pair was converted.
2773 for leaf in node.leaves():
2774 previous_consumed = 0
2775 for comment in list_comments(leaf.prefix, is_endmarker=False):
2776 if comment.value in FMT_OFF:
2777 # We only want standalone comments. If there's no previous leaf or
2778 # the previous leaf is indentation, it's a standalone comment in
2780 if comment.type != STANDALONE_COMMENT:
2781 prev = preceding_leaf(leaf)
2782 if prev and prev.type not in WHITESPACE:
2785 ignored_nodes = list(generate_ignored_nodes(leaf))
2786 if not ignored_nodes:
2789 first = ignored_nodes[0] # Can be a container node with the `leaf`.
2790 parent = first.parent
2791 prefix = first.prefix
2792 first.prefix = prefix[comment.consumed :]
2794 comment.value + "\n" + "".join(str(n) for n in ignored_nodes)
2796 if hidden_value.endswith("\n"):
2797 # That happens when one of the `ignored_nodes` ended with a NEWLINE
2798 # leaf (possibly followed by a DEDENT).
2799 hidden_value = hidden_value[:-1]
2801 for ignored in ignored_nodes:
2802 index = ignored.remove()
2803 if first_idx is None:
2805 assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
2806 assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
2807 parent.insert_child(
2812 prefix=prefix[:previous_consumed] + "\n" * comment.newlines,
2817 previous_consumed = comment.consumed
2822 def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
2823 """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
2825 Stops at the end of the block.
2827 container: Optional[LN] = container_of(leaf)
2828 while container is not None and container.type != token.ENDMARKER:
2829 for comment in list_comments(container.prefix, is_endmarker=False):
2830 if comment.value in FMT_ON:
2835 container = container.next_sibling
2838 def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
2839 """If it's safe, make the parens in the atom `node` invisible, recursively.
2841 Returns whether the node should itself be wrapped in invisible parentheses.
2845 node.type != syms.atom
2846 or is_empty_tuple(node)
2847 or is_one_tuple(node)
2849 or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
2853 first = node.children[0]
2854 last = node.children[-1]
2855 if first.type == token.LPAR and last.type == token.RPAR:
2856 # make parentheses invisible
2857 first.value = "" # type: ignore
2858 last.value = "" # type: ignore
2859 if len(node.children) > 1:
2860 maybe_make_parens_invisible_in_atom(node.children[1])
2866 def is_empty_tuple(node: LN) -> bool:
2867 """Return True if `node` holds an empty tuple."""
2869 node.type == syms.atom
2870 and len(node.children) == 2
2871 and node.children[0].type == token.LPAR
2872 and node.children[1].type == token.RPAR
2876 def is_one_tuple(node: LN) -> bool:
2877 """Return True if `node` holds a tuple with one element, with or without parens."""
2878 if node.type == syms.atom:
2879 if len(node.children) != 3:
2882 lpar, gexp, rpar = node.children
2884 lpar.type == token.LPAR
2885 and gexp.type == syms.testlist_gexp
2886 and rpar.type == token.RPAR
2890 return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
2893 node.type in IMPLICIT_TUPLE
2894 and len(node.children) == 2
2895 and node.children[1].type == token.COMMA
2899 def is_yield(node: LN) -> bool:
2900 """Return True if `node` holds a `yield` or `yield from` expression."""
2901 if node.type == syms.yield_expr:
2904 if node.type == token.NAME and node.value == "yield": # type: ignore
2907 if node.type != syms.atom:
2910 if len(node.children) != 3:
2913 lpar, expr, rpar = node.children
2914 if lpar.type == token.LPAR and rpar.type == token.RPAR:
2915 return is_yield(expr)
2920 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
2921 """Return True if `leaf` is a star or double star in a vararg or kwarg.
2923 If `within` includes VARARGS_PARENTS, this applies to function signatures.
2924 If `within` includes UNPACKING_PARENTS, it applies to right hand-side
2925 extended iterable unpacking (PEP 3132) and additional unpacking
2926 generalizations (PEP 448).
2928 if leaf.type not in STARS or not leaf.parent:
2932 if p.type == syms.star_expr:
2933 # Star expressions are also used as assignment targets in extended
2934 # iterable unpacking (PEP 3132). See what its parent is instead.
2940 return p.type in within
2943 def is_multiline_string(leaf: Leaf) -> bool:
2944 """Return True if `leaf` is a multiline string that actually spans many lines."""
2945 value = leaf.value.lstrip("furbFURB")
2946 return value[:3] in {'"""', "'''"} and "\n" in value
2949 def is_stub_suite(node: Node) -> bool:
2950 """Return True if `node` is a suite with a stub body."""
2952 len(node.children) != 4
2953 or node.children[0].type != token.NEWLINE
2954 or node.children[1].type != token.INDENT
2955 or node.children[3].type != token.DEDENT
2959 return is_stub_body(node.children[2])
2962 def is_stub_body(node: LN) -> bool:
2963 """Return True if `node` is a simple statement containing an ellipsis."""
2964 if not isinstance(node, Node) or node.type != syms.simple_stmt:
2967 if len(node.children) != 2:
2970 child = node.children[0]
2972 child.type == syms.atom
2973 and len(child.children) == 3
2974 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
2978 def max_delimiter_priority_in_atom(node: LN) -> int:
2979 """Return maximum delimiter priority inside `node`.
2981 This is specific to atoms with contents contained in a pair of parentheses.
2982 If `node` isn't an atom or there are no enclosing parentheses, returns 0.
2984 if node.type != syms.atom:
2987 first = node.children[0]
2988 last = node.children[-1]
2989 if not (first.type == token.LPAR and last.type == token.RPAR):
2992 bt = BracketTracker()
2993 for c in node.children[1:-1]:
2994 if isinstance(c, Leaf):
2997 for leaf in c.leaves():
3000 return bt.max_delimiter_priority()
3006 def ensure_visible(leaf: Leaf) -> None:
3007 """Make sure parentheses are visible.
3009 They could be invisible as part of some statements (see
3010 :func:`normalize_invible_parens` and :func:`visit_import_from`).
3012 if leaf.type == token.LPAR:
3014 elif leaf.type == token.RPAR:
3018 def should_explode(line: Line, opening_bracket: Leaf) -> bool:
3019 """Should `line` immediately be split with `delimiter_split()` after RHS?"""
3022 opening_bracket.parent
3023 and opening_bracket.parent.type in {syms.atom, syms.import_from}
3024 and opening_bracket.value in "[{("
3029 last_leaf = line.leaves[-1]
3030 exclude = {id(last_leaf)} if last_leaf.type == token.COMMA else set()
3031 max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
3032 except (IndexError, ValueError):
3035 return max_priority == COMMA_PRIORITY
3038 def get_features_used(node: Node) -> Set[Feature]:
3039 """Return a set of (relatively) new Python features used in this file.
3041 Currently looking for:
3043 - underscores in numeric literals; and
3044 - trailing commas after * or ** in function signatures and calls.
3046 features: Set[Feature] = set()
3047 for n in node.pre_order():
3048 if n.type == token.STRING:
3049 value_head = n.value[:2] # type: ignore
3050 if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
3051 features.add(Feature.F_STRINGS)
3053 elif n.type == token.NUMBER:
3054 if "_" in n.value: # type: ignore
3055 features.add(Feature.NUMERIC_UNDERSCORES)
3058 n.type in {syms.typedargslist, syms.arglist}
3060 and n.children[-1].type == token.COMMA
3062 for ch in n.children:
3063 if ch.type in STARS:
3064 features.add(Feature.TRAILING_COMMA)
3066 if ch.type == syms.argument:
3067 for argch in ch.children:
3068 if argch.type in STARS:
3069 features.add(Feature.TRAILING_COMMA)
3074 def detect_target_versions(node: Node) -> Set[TargetVersion]:
3075 """Detect the version to target based on the nodes used."""
3076 features = get_features_used(node)
3078 version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
3082 def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
3083 """Generate sets of closing bracket IDs that should be omitted in a RHS.
3085 Brackets can be omitted if the entire trailer up to and including
3086 a preceding closing bracket fits in one line.
3088 Yielded sets are cumulative (contain results of previous yields, too). First
3092 omit: Set[LeafID] = set()
3095 length = 4 * line.depth
3096 opening_bracket = None
3097 closing_bracket = None
3098 inner_brackets: Set[LeafID] = set()
3099 for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
3100 length += leaf_length
3101 if length > line_length:
3104 has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
3105 if leaf.type == STANDALONE_COMMENT or has_inline_comment:
3109 if leaf is opening_bracket:
3110 opening_bracket = None
3111 elif leaf.type in CLOSING_BRACKETS:
3112 inner_brackets.add(id(leaf))
3113 elif leaf.type in CLOSING_BRACKETS:
3114 if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
3115 # Empty brackets would fail a split so treat them as "inner"
3116 # brackets (e.g. only add them to the `omit` set if another
3117 # pair of brackets was good enough.
3118 inner_brackets.add(id(leaf))
3122 omit.add(id(closing_bracket))
3123 omit.update(inner_brackets)
3124 inner_brackets.clear()
3128 opening_bracket = leaf.opening_bracket
3129 closing_bracket = leaf
3132 def get_future_imports(node: Node) -> Set[str]:
3133 """Return a set of __future__ imports in the file."""
3134 imports: Set[str] = set()
3136 def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
3137 for child in children:
3138 if isinstance(child, Leaf):
3139 if child.type == token.NAME:
3141 elif child.type == syms.import_as_name:
3142 orig_name = child.children[0]
3143 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
3144 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
3145 yield orig_name.value
3146 elif child.type == syms.import_as_names:
3147 yield from get_imports_from_children(child.children)
3149 assert False, "Invalid syntax parsing imports"
3151 for child in node.children:
3152 if child.type != syms.simple_stmt:
3154 first_child = child.children[0]
3155 if isinstance(first_child, Leaf):
3156 # Continue looking if we see a docstring; otherwise stop.
3158 len(child.children) == 2
3159 and first_child.type == token.STRING
3160 and child.children[1].type == token.NEWLINE
3165 elif first_child.type == syms.import_from:
3166 module_name = first_child.children[1]
3167 if not isinstance(module_name, Leaf) or module_name.value != "__future__":
3169 imports |= set(get_imports_from_children(first_child.children[3:]))
3175 def gen_python_files_in_dir(
3178 include: Pattern[str],
3179 exclude: Pattern[str],
3181 ) -> Iterator[Path]:
3182 """Generate all files under `path` whose paths are not excluded by the
3183 `exclude` regex, but are included by the `include` regex.
3185 Symbolic links pointing outside of the `root` directory are ignored.
3187 `report` is where output about exclusions goes.
3189 assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
3190 for child in path.iterdir():
3192 normalized_path = "/" + child.resolve().relative_to(root).as_posix()
3194 if child.is_symlink():
3195 report.path_ignored(
3196 child, f"is a symbolic link that points outside {root}"
3203 normalized_path += "/"
3204 exclude_match = exclude.search(normalized_path)
3205 if exclude_match and exclude_match.group(0):
3206 report.path_ignored(child, f"matches the --exclude regular expression")
3210 yield from gen_python_files_in_dir(child, root, include, exclude, report)
3212 elif child.is_file():
3213 include_match = include.search(normalized_path)
3219 def find_project_root(srcs: Iterable[str]) -> Path:
3220 """Return a directory containing .git, .hg, or pyproject.toml.
3222 That directory can be one of the directories passed in `srcs` or their
3225 If no directory in the tree contains a marker that would specify it's the
3226 project root, the root of the file system is returned.
3229 return Path("/").resolve()
3231 common_base = min(Path(src).resolve() for src in srcs)
3232 if common_base.is_dir():
3233 # Append a fake file so `parents` below returns `common_base_dir`, too.
3234 common_base /= "fake-file"
3235 for directory in common_base.parents:
3236 if (directory / ".git").is_dir():
3239 if (directory / ".hg").is_dir():
3242 if (directory / "pyproject.toml").is_file():
3250 """Provides a reformatting counter. Can be rendered with `str(report)`."""
3254 verbose: bool = False
3255 change_count: int = 0
3257 failure_count: int = 0
3259 def done(self, src: Path, changed: Changed) -> None:
3260 """Increment the counter for successful reformatting. Write out a message."""
3261 if changed is Changed.YES:
3262 reformatted = "would reformat" if self.check else "reformatted"
3263 if self.verbose or not self.quiet:
3264 out(f"{reformatted} {src}")
3265 self.change_count += 1
3268 if changed is Changed.NO:
3269 msg = f"{src} already well formatted, good job."
3271 msg = f"{src} wasn't modified on disk since last run."
3272 out(msg, bold=False)
3273 self.same_count += 1
3275 def failed(self, src: Path, message: str) -> None:
3276 """Increment the counter for failed reformatting. Write out a message."""
3277 err(f"error: cannot format {src}: {message}")
3278 self.failure_count += 1
3280 def path_ignored(self, path: Path, message: str) -> None:
3282 out(f"{path} ignored: {message}", bold=False)
3285 def return_code(self) -> int:
3286 """Return the exit code that the app should use.
3288 This considers the current state of changed files and failures:
3289 - if there were any failures, return 123;
3290 - if any files were changed and --check is being used, return 1;
3291 - otherwise return 0.
3293 # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
3294 # 126 we have special return codes reserved by the shell.
3295 if self.failure_count:
3298 elif self.change_count and self.check:
3303 def __str__(self) -> str:
3304 """Render a color report of the current state.
3306 Use `click.unstyle` to remove colors.
3309 reformatted = "would be reformatted"
3310 unchanged = "would be left unchanged"
3311 failed = "would fail to reformat"
3313 reformatted = "reformatted"
3314 unchanged = "left unchanged"
3315 failed = "failed to reformat"
3317 if self.change_count:
3318 s = "s" if self.change_count > 1 else ""
3320 click.style(f"{self.change_count} file{s} {reformatted}", bold=True)
3323 s = "s" if self.same_count > 1 else ""
3324 report.append(f"{self.same_count} file{s} {unchanged}")
3325 if self.failure_count:
3326 s = "s" if self.failure_count > 1 else ""
3328 click.style(f"{self.failure_count} file{s} {failed}", fg="red")
3330 return ", ".join(report) + "."
3333 def assert_equivalent(src: str, dst: str) -> None:
3334 """Raise AssertionError if `src` and `dst` aren't equivalent."""
3339 def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
3340 """Simple visitor generating strings to compare ASTs by content."""
3341 yield f"{' ' * depth}{node.__class__.__name__}("
3343 for field in sorted(node._fields):
3345 value = getattr(node, field)
3346 except AttributeError:
3349 yield f"{' ' * (depth+1)}{field}="
3351 if isinstance(value, list):
3353 # Ignore nested tuples within del statements, because we may insert
3354 # parentheses and they change the AST.
3357 and isinstance(node, ast.Delete)
3358 and isinstance(item, ast.Tuple)
3360 for item in item.elts:
3361 yield from _v(item, depth + 2)
3362 elif isinstance(item, ast.AST):
3363 yield from _v(item, depth + 2)
3365 elif isinstance(value, ast.AST):
3366 yield from _v(value, depth + 2)
3369 yield f"{' ' * (depth+2)}{value!r}, # {value.__class__.__name__}"
3371 yield f"{' ' * depth}) # /{node.__class__.__name__}"
3374 src_ast = ast.parse(src)
3375 except Exception as exc:
3376 major, minor = sys.version_info[:2]
3377 raise AssertionError(
3378 f"cannot use --safe with this file; failed to parse source file "
3379 f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
3380 f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
3384 dst_ast = ast.parse(dst)
3385 except Exception as exc:
3386 log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
3387 raise AssertionError(
3388 f"INTERNAL ERROR: Black produced invalid code: {exc}. "
3389 f"Please report a bug on https://github.com/ambv/black/issues. "
3390 f"This invalid output might be helpful: {log}"
3393 src_ast_str = "\n".join(_v(src_ast))
3394 dst_ast_str = "\n".join(_v(dst_ast))
3395 if src_ast_str != dst_ast_str:
3396 log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
3397 raise AssertionError(
3398 f"INTERNAL ERROR: Black produced code that is not equivalent to "
3400 f"Please report a bug on https://github.com/ambv/black/issues. "
3401 f"This diff might be helpful: {log}"
3405 def assert_stable(src: str, dst: str, mode: FileMode) -> None:
3406 """Raise AssertionError if `dst` reformats differently the second time."""
3407 newdst = format_str(dst, mode=mode)
3410 diff(src, dst, "source", "first pass"),
3411 diff(dst, newdst, "first pass", "second pass"),
3413 raise AssertionError(
3414 f"INTERNAL ERROR: Black produced different code on the second pass "
3415 f"of the formatter. "
3416 f"Please report a bug on https://github.com/ambv/black/issues. "
3417 f"This diff might be helpful: {log}"
3421 def dump_to_file(*output: str) -> str:
3422 """Dump `output` to a temporary file. Return path to the file."""
3425 with tempfile.NamedTemporaryFile(
3426 mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8"
3428 for lines in output:
3430 if lines and lines[-1] != "\n":
3435 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
3436 """Return a unified diff string between strings `a` and `b`."""
3439 a_lines = [line + "\n" for line in a.split("\n")]
3440 b_lines = [line + "\n" for line in b.split("\n")]
3442 difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5)
3446 def cancel(tasks: Iterable[asyncio.Task]) -> None:
3447 """asyncio signal handler that cancels all `tasks` and reports to stderr."""
3453 def shutdown(loop: BaseEventLoop) -> None:
3454 """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
3456 if sys.version_info[:2] >= (3, 7):
3457 all_tasks = asyncio.all_tasks
3459 all_tasks = asyncio.Task.all_tasks
3460 # This part is borrowed from asyncio/runners.py in Python 3.7b2.
3461 to_cancel = [task for task in all_tasks(loop) if not task.done()]
3465 for task in to_cancel:
3467 loop.run_until_complete(
3468 asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)
3471 # `concurrent.futures.Future` objects cannot be cancelled once they
3472 # are already running. There might be some when the `shutdown()` happened.
3473 # Silence their logger's spew about the event loop being closed.
3474 cf_logger = logging.getLogger("concurrent.futures")
3475 cf_logger.setLevel(logging.CRITICAL)
3479 def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
3480 """Replace `regex` with `replacement` twice on `original`.
3482 This is used by string normalization to perform replaces on
3483 overlapping matches.
3485 return regex.sub(replacement, regex.sub(replacement, original))
3488 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
3489 """Compile a regular expression string in `regex`.
3491 If it contains newlines, use verbose mode.
3494 regex = "(?x)" + regex
3495 return re.compile(regex)
3498 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
3499 """Like `reversed(enumerate(sequence))` if that were possible."""
3500 index = len(sequence) - 1
3501 for element in reversed(sequence):
3502 yield (index, element)
3506 def enumerate_with_length(
3507 line: Line, reversed: bool = False
3508 ) -> Iterator[Tuple[Index, Leaf, int]]:
3509 """Return an enumeration of leaves with their length.
3511 Stops prematurely on multiline strings and standalone comments.
3514 Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]],
3515 enumerate_reversed if reversed else enumerate,
3517 for index, leaf in op(line.leaves):
3518 length = len(leaf.prefix) + len(leaf.value)
3519 if "\n" in leaf.value:
3520 return # Multiline strings, we can't continue.
3522 comment: Optional[Leaf]
3523 for comment in line.comments_after(leaf):
3524 length += len(comment.value)
3526 yield index, leaf, length
3529 def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
3530 """Return True if `line` is no longer than `line_length`.
3532 Uses the provided `line_str` rendering, if any, otherwise computes a new one.
3535 line_str = str(line).strip("\n")
3537 len(line_str) <= line_length
3538 and "\n" not in line_str # multiline strings
3539 and not line.contains_standalone_comments()
3543 def can_be_split(line: Line) -> bool:
3544 """Return False if the line cannot be split *for sure*.
3546 This is not an exhaustive search but a cheap heuristic that we can use to
3547 avoid some unfortunate formattings (mostly around wrapping unsplittable code
3548 in unnecessary parentheses).
3550 leaves = line.leaves
3554 if leaves[0].type == token.STRING and leaves[1].type == token.DOT:
3558 for leaf in leaves[-2::-1]:
3559 if leaf.type in OPENING_BRACKETS:
3560 if next.type not in CLOSING_BRACKETS:
3564 elif leaf.type == token.DOT:
3566 elif leaf.type == token.NAME:
3567 if not (next.type == token.DOT or next.type in OPENING_BRACKETS):
3570 elif leaf.type not in CLOSING_BRACKETS:
3573 if dot_count > 1 and call_count > 1:
3579 def can_omit_invisible_parens(line: Line, line_length: int) -> bool:
3580 """Does `line` have a shape safe to reformat without optional parens around it?
3582 Returns True for only a subset of potentially nice looking formattings but
3583 the point is to not return false positives that end up producing lines that
3586 bt = line.bracket_tracker
3587 if not bt.delimiters:
3588 # Without delimiters the optional parentheses are useless.
3591 max_priority = bt.max_delimiter_priority()
3592 if bt.delimiter_count_with_priority(max_priority) > 1:
3593 # With more than one delimiter of a kind the optional parentheses read better.
3596 if max_priority == DOT_PRIORITY:
3597 # A single stranded method call doesn't require optional parentheses.
3600 assert len(line.leaves) >= 2, "Stranded delimiter"
3602 first = line.leaves[0]
3603 second = line.leaves[1]
3604 penultimate = line.leaves[-2]
3605 last = line.leaves[-1]
3607 # With a single delimiter, omit if the expression starts or ends with
3609 if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS:
3611 length = 4 * line.depth
3612 for _index, leaf, leaf_length in enumerate_with_length(line):
3613 if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first:
3616 length += leaf_length
3617 if length > line_length:
3620 if leaf.type in OPENING_BRACKETS:
3621 # There are brackets we can further split on.
3625 # checked the entire string and line length wasn't exceeded
3626 if len(line.leaves) == _index + 1:
3629 # Note: we are not returning False here because a line might have *both*
3630 # a leading opening bracket and a trailing closing bracket. If the
3631 # opening bracket doesn't match our rule, maybe the closing will.
3634 last.type == token.RPAR
3635 or last.type == token.RBRACE
3637 # don't use indexing for omitting optional parentheses;
3639 last.type == token.RSQB
3641 and last.parent.type != syms.trailer
3644 if penultimate.type in OPENING_BRACKETS:
3645 # Empty brackets don't help.
3648 if is_multiline_string(first):
3649 # Additional wrapping of a multiline string in this situation is
3653 length = 4 * line.depth
3654 seen_other_brackets = False
3655 for _index, leaf, leaf_length in enumerate_with_length(line):
3656 length += leaf_length
3657 if leaf is last.opening_bracket:
3658 if seen_other_brackets or length <= line_length:
3661 elif leaf.type in OPENING_BRACKETS:
3662 # There are brackets we can further split on.
3663 seen_other_brackets = True
3668 def get_cache_file(mode: FileMode) -> Path:
3669 return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle"
3672 def read_cache(mode: FileMode) -> Cache:
3673 """Read the cache if it exists and is well formed.
3675 If it is not well formed, the call to write_cache later should resolve the issue.
3677 cache_file = get_cache_file(mode)
3678 if not cache_file.exists():
3681 with cache_file.open("rb") as fobj:
3683 cache: Cache = pickle.load(fobj)
3684 except pickle.UnpicklingError:
3690 def get_cache_info(path: Path) -> CacheInfo:
3691 """Return the information used to check if a file is already formatted or not."""
3693 return stat.st_mtime, stat.st_size
3696 def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]:
3697 """Split an iterable of paths in `sources` into two sets.
3699 The first contains paths of files that modified on disk or are not in the
3700 cache. The other contains paths to non-modified files.
3702 todo, done = set(), set()
3705 if cache.get(src) != get_cache_info(src):
3712 def write_cache(cache: Cache, sources: Iterable[Path], mode: FileMode) -> None:
3713 """Update the cache file."""
3714 cache_file = get_cache_file(mode)
3716 CACHE_DIR.mkdir(parents=True, exist_ok=True)
3717 new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
3718 with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
3719 pickle.dump(new_cache, f, protocol=pickle.HIGHEST_PROTOCOL)
3720 os.replace(f.name, cache_file)
3725 def patch_click() -> None:
3726 """Make Click not crash.
3728 On certain misconfigured environments, Python 3 selects the ASCII encoding as the
3729 default which restricts paths that it can access during the lifetime of the
3730 application. Click refuses to work in this scenario by raising a RuntimeError.
3732 In case of Black the likelihood that non-ASCII characters are going to be used in
3733 file paths is minimal since it's Python source code. Moreover, this crash was
3734 spurious on Python 3.7 thanks to PEP 538 and PEP 540.
3737 from click import core
3738 from click import _unicodefun # type: ignore
3739 except ModuleNotFoundError:
3742 for module in (core, _unicodefun):
3743 if hasattr(module, "_verify_python3_env"):
3744 module._verify_python3_env = lambda: None
3747 def patched_main() -> None:
3753 if __name__ == "__main__":