All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 from json.decoder import JSONDecodeError
4 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
5 from contextlib import contextmanager
6 from datetime import datetime
9 from multiprocessing import Manager, freeze_support
11 from pathlib import Path
12 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
35 from click.core import ParameterSource
36 from dataclasses import replace
37 from mypy_extensions import mypyc_attr
39 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
40 from black.const import STDIN_PLACEHOLDER
41 from black.nodes import STARS, syms, is_simple_decorator_expression
42 from black.nodes import is_string_token
43 from black.lines import Line, EmptyLineTracker
44 from black.linegen import transform_line, LineGenerator, LN
45 from black.comments import normalize_fmt_off
46 from black.mode import FUTURE_FLAG_TO_FEATURE, Mode, TargetVersion
47 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
48 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
49 from black.concurrency import cancel, shutdown, maybe_install_uvloop
50 from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err
51 from black.report import Report, Changed, NothingChanged
52 from black.files import (
56 find_user_pyproject_toml,
58 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
59 from black.files import wrap_stream_for_windows
60 from black.parsing import InvalidInput # noqa F401
61 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
62 from black.handle_ipynb_magics import (
65 remove_trailing_semicolon,
66 put_trailing_semicolon_back,
69 jupyter_dependencies_are_installed,
74 from blib2to3.pytree import Node, Leaf
75 from blib2to3.pgen2 import token
77 from _black_version import version as __version__
79 COMPILED = Path(__file__).suffix in (".pyd", ".so")
87 class WriteBack(Enum):
95 def from_configuration(
96 cls, *, check: bool, diff: bool, color: bool = False
98 if check and not diff:
102 return cls.COLOR_DIFF
104 return cls.DIFF if diff else cls.YES
107 # Legacy name, left for integrations.
110 DEFAULT_WORKERS = os.cpu_count()
113 def read_pyproject_toml(
114 ctx: click.Context, param: click.Parameter, value: Optional[str]
116 """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
118 Returns the path to a successfully found and read configuration file, None
122 value = find_pyproject_toml(ctx.params.get("src", ()))
127 config = parse_pyproject_toml(value)
128 except (OSError, ValueError) as e:
129 raise click.FileError(
130 filename=value, hint=f"Error reading configuration file: {e}"
136 # Sanitize the values to be Click friendly. For more information please see:
137 # https://github.com/psf/black/issues/1458
138 # https://github.com/pallets/click/issues/1567
140 k: str(v) if not isinstance(v, (list, dict)) else v
141 for k, v in config.items()
144 target_version = config.get("target_version")
145 if target_version is not None and not isinstance(target_version, list):
146 raise click.BadOptionUsage(
147 "target-version", "Config key target-version must be a list"
150 default_map: Dict[str, Any] = {}
152 default_map.update(ctx.default_map)
153 default_map.update(config)
155 ctx.default_map = default_map
159 def target_version_option_callback(
160 c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
161 ) -> List[TargetVersion]:
162 """Compute the target versions from a --target-version flag.
164 This is its own function because mypy couldn't infer the type correctly
165 when it was a lambda, causing mypyc trouble.
167 return [TargetVersion[val.upper()] for val in v]
170 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
171 """Compile a regular expression string in `regex`.
173 If it contains newlines, use verbose mode.
176 regex = "(?x)" + regex
177 compiled: Pattern[str] = re.compile(regex)
183 param: click.Parameter,
184 value: Optional[str],
185 ) -> Optional[Pattern[str]]:
187 return re_compile_maybe_verbose(value) if value is not None else None
188 except re.error as e:
189 raise click.BadParameter(f"Not a valid regular expression: {e}") from None
193 context_settings={"help_option_names": ["-h", "--help"]},
194 # While Click does set this field automatically using the docstring, mypyc
195 # (annoyingly) strips 'em so we need to set it here too.
196 help="The uncompromising code formatter.",
198 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
203 default=DEFAULT_LINE_LENGTH,
204 help="How many characters per line to allow.",
210 type=click.Choice([v.name.lower() for v in TargetVersion]),
211 callback=target_version_option_callback,
214 "Python versions that should be supported by Black's output. [default: per-file"
222 "Format all input files like typing stubs regardless of file extension (useful"
223 " when piping source on standard input)."
230 "Format all input files like Jupyter Notebooks regardless of file extension "
231 "(useful when piping source on standard input)."
235 "--python-cell-magics",
238 "When processing Jupyter Notebooks, add the given magic to the list"
239 f" of known python-magics ({', '.join(PYTHON_CELL_MAGICS)})."
240 " Useful for formatting cells with custom python magics."
246 "--skip-string-normalization",
248 help="Don't normalize string quotes or prefixes.",
252 "--skip-magic-trailing-comma",
254 help="Don't use trailing commas as a reason to split lines.",
257 "--experimental-string-processing",
260 help="(DEPRECATED and now included in --preview) Normalize string literals.",
266 "Enable potentially disruptive style changes that may be added to Black's main"
267 " functionality in the next major release."
274 "Don't write the files back, just return the status. Return code 0 means"
275 " nothing would change. Return code 1 means some files would be reformatted."
276 " Return code 123 means there was an internal error."
282 help="Don't write the files back, just output a diff for each file on stdout.",
285 "--color/--no-color",
287 help="Show colored diff. Only applies when `--diff` is given.",
292 help="If --fast given, skip temporary sanity checks. [default: --safe]",
295 "--required-version",
298 "Require a specific version of Black to be running (useful for unifying results"
299 " across many environments e.g. with a pyproject.toml file). It can be"
300 " either a major version number or an exact version."
306 default=DEFAULT_INCLUDES,
307 callback=validate_regex,
309 "A regular expression that matches files and directories that should be"
310 " included on recursive searches. An empty value means all files are included"
311 " regardless of the name. Use forward slashes for directories on all platforms"
312 " (Windows, too). Exclusions are calculated first, inclusions later."
319 callback=validate_regex,
321 "A regular expression that matches files and directories that should be"
322 " excluded on recursive searches. An empty value means no paths are excluded."
323 " Use forward slashes for directories on all platforms (Windows, too)."
324 " Exclusions are calculated first, inclusions later. [default:"
325 f" {DEFAULT_EXCLUDES}]"
332 callback=validate_regex,
334 "Like --exclude, but adds additional files and directories on top of the"
335 " excluded ones. (Useful if you simply want to add to the default)"
341 callback=validate_regex,
343 "Like --exclude, but files and directories matching this regex will be "
344 "excluded even when they are passed explicitly as arguments."
351 "The name of the file when passing it through stdin. Useful to make "
352 "sure Black will respect --force-exclude option on some "
353 "editors that rely on using stdin."
359 type=click.IntRange(min=1),
360 default=DEFAULT_WORKERS,
362 help="Number of parallel workers",
369 "Don't emit non-error messages to stderr. Errors are still emitted; silence"
370 " those with 2>/dev/null."
378 "Also emit messages to stderr about files that were not changed or were ignored"
379 " due to exclusion patterns."
382 @click.version_option(
384 message=f"%(prog)s, %(version)s (compiled: {'yes' if COMPILED else 'no'})",
390 exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
406 callback=read_pyproject_toml,
407 help="Read configuration from FILE path.",
410 def main( # noqa: C901
414 target_version: List[TargetVersion],
421 python_cell_magics: Sequence[str],
422 skip_string_normalization: bool,
423 skip_magic_trailing_comma: bool,
424 experimental_string_processing: bool,
428 required_version: Optional[str],
429 include: Pattern[str],
430 exclude: Optional[Pattern[str]],
431 extend_exclude: Optional[Pattern[str]],
432 force_exclude: Optional[Pattern[str]],
433 stdin_filename: Optional[str],
435 src: Tuple[str, ...],
436 config: Optional[str],
438 """The uncompromising code formatter."""
439 ctx.ensure_object(dict)
441 if src and code is not None:
444 + "\n\n'SRC' and 'code' cannot be passed simultaneously."
447 if not src and code is None:
448 out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.")
451 root, method = find_project_root(src) if code is None else (None, None)
452 ctx.obj["root"] = root
457 f"Identified `{root}` as project root containing a {method}.",
462 (normalize_path_maybe_ignore(Path(source), root), source)
465 srcs_string = ", ".join(
469 else f'\033[31m"{source} (skipping - invalid)"\033[34m'
470 for _norm, source in normalized
473 out(f"Sources to be formatted: {srcs_string}", fg="blue")
476 config_source = ctx.get_parameter_source("config")
477 user_level_config = str(find_user_pyproject_toml())
478 if config == user_level_config:
480 f"Using configuration from user-level config at "
481 f"'{user_level_config}'.",
484 elif config_source in (
485 ParameterSource.DEFAULT,
486 ParameterSource.DEFAULT_MAP,
488 out("Using configuration from project root.", fg="blue")
490 out(f"Using configuration in '{config}'.", fg="blue")
492 error_msg = "Oh no! 💥 💔 💥"
495 and required_version != __version__
496 and required_version != __version__.split(".")[0]
499 f"{error_msg} The required version `{required_version}` does not match"
500 f" the running version `{__version__}`!"
504 err("Cannot pass both `pyi` and `ipynb` flags!")
507 write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
509 versions = set(target_version)
511 # We'll autodetect later.
514 target_versions=versions,
515 line_length=line_length,
518 string_normalization=not skip_string_normalization,
519 magic_trailing_comma=not skip_magic_trailing_comma,
520 experimental_string_processing=experimental_string_processing,
522 python_cell_magics=set(python_cell_magics),
526 # Run in quiet mode by default with -c; the extra output isn't useful.
527 # You can still pass -v to get verbose output.
530 report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
534 content=code, fast=fast, write_back=write_back, mode=mode, report=report
538 sources = get_sources(
545 extend_exclude=extend_exclude,
546 force_exclude=force_exclude,
548 stdin_filename=stdin_filename,
550 except GitWildMatchPatternError:
555 "No Python files are present to be formatted. Nothing to do 😴",
561 if len(sources) == 1:
565 write_back=write_back,
573 write_back=write_back,
579 if verbose or not quiet:
580 if code is None and (verbose or report.change_count or report.failure_count):
582 out(error_msg if report.return_code else "All done! ✨ 🍰 ✨")
584 click.echo(str(report), err=True)
585 ctx.exit(report.return_code)
591 src: Tuple[str, ...],
594 include: Pattern[str],
595 exclude: Optional[Pattern[str]],
596 extend_exclude: Optional[Pattern[str]],
597 force_exclude: Optional[Pattern[str]],
599 stdin_filename: Optional[str],
601 """Compute the set of files to be formatted."""
602 sources: Set[Path] = set()
605 exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
606 gitignore = get_gitignore(ctx.obj["root"])
611 if s == "-" and stdin_filename:
612 p = Path(stdin_filename)
618 if is_stdin or p.is_file():
619 normalized_path = normalize_path_maybe_ignore(p, ctx.obj["root"], report)
620 if normalized_path is None:
623 normalized_path = "/" + normalized_path
624 # Hard-exclude any files that matches the `--force-exclude` regex.
626 force_exclude_match = force_exclude.search(normalized_path)
628 force_exclude_match = None
629 if force_exclude_match and force_exclude_match.group(0):
630 report.path_ignored(p, "matches the --force-exclude regular expression")
634 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
636 if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
637 verbose=verbose, quiet=quiet
660 err(f"invalid path: {s}")
665 src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
668 Exit if there is no `src` provided for formatting
671 if verbose or not quiet:
677 content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report
680 Reformat and print out `content` without spawning child processes.
681 Similar to `reformat_one`, but for string content.
683 `fast`, `write_back`, and `mode` options are passed to
684 :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
686 path = Path("<string>")
689 if format_stdin_to_stdout(
690 content=content, fast=fast, write_back=write_back, mode=mode
692 changed = Changed.YES
693 report.done(path, changed)
694 except Exception as exc:
696 traceback.print_exc()
697 report.failed(path, str(exc))
701 src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
703 """Reformat a single file under `src` without spawning child processes.
705 `fast`, `write_back`, and `mode` options are passed to
706 :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
713 elif str(src).startswith(STDIN_PLACEHOLDER):
715 # Use the original name again in case we want to print something
717 src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
722 if src.suffix == ".pyi":
723 mode = replace(mode, is_pyi=True)
724 elif src.suffix == ".ipynb":
725 mode = replace(mode, is_ipynb=True)
726 if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
727 changed = Changed.YES
730 if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
731 cache = read_cache(mode)
732 res_src = src.resolve()
733 res_src_s = str(res_src)
734 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
735 changed = Changed.CACHED
736 if changed is not Changed.CACHED and format_file_in_place(
737 src, fast=fast, write_back=write_back, mode=mode
739 changed = Changed.YES
740 if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
741 write_back is WriteBack.CHECK and changed is Changed.NO
743 write_cache(cache, [src], mode)
744 report.done(src, changed)
745 except Exception as exc:
747 traceback.print_exc()
748 report.failed(src, str(exc))
751 # diff-shades depends on being to monkeypatch this function to operate. I know it's
752 # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26
753 @mypyc_attr(patchable=True)
757 write_back: WriteBack,
760 workers: Optional[int],
762 """Reformat multiple files using a ProcessPoolExecutor."""
764 loop = asyncio.get_event_loop()
765 worker_count = workers if workers is not None else DEFAULT_WORKERS
766 if sys.platform == "win32":
767 # Work around https://bugs.python.org/issue26903
768 assert worker_count is not None
769 worker_count = min(worker_count, 60)
771 executor = ProcessPoolExecutor(max_workers=worker_count)
772 except (ImportError, NotImplementedError, OSError):
773 # we arrive here if the underlying system does not support multi-processing
774 # like in AWS Lambda or Termux, in which case we gracefully fallback to
775 # a ThreadPoolExecutor with just a single worker (more workers would not do us
776 # any good due to the Global Interpreter Lock)
777 executor = ThreadPoolExecutor(max_workers=1)
780 loop.run_until_complete(
784 write_back=write_back,
793 if executor is not None:
797 async def schedule_formatting(
800 write_back: WriteBack,
803 loop: asyncio.AbstractEventLoop,
806 """Run formatting of `sources` in parallel using the provided `executor`.
808 (Use ProcessPoolExecutors for actual parallelism.)
810 `write_back`, `fast`, and `mode` options are passed to
811 :func:`format_file_in_place`.
814 if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
815 cache = read_cache(mode)
816 sources, cached = filter_cached(cache, sources)
817 for src in sorted(cached):
818 report.done(src, Changed.CACHED)
823 sources_to_cache = []
825 if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
826 # For diff output, we need locks to ensure we don't interleave output
827 # from different processes.
829 lock = manager.Lock()
831 asyncio.ensure_future(
832 loop.run_in_executor(
833 executor, format_file_in_place, src, fast, mode, write_back, lock
836 for src in sorted(sources)
838 pending = tasks.keys()
840 loop.add_signal_handler(signal.SIGINT, cancel, pending)
841 loop.add_signal_handler(signal.SIGTERM, cancel, pending)
842 except NotImplementedError:
843 # There are no good alternatives for these on Windows.
846 done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
848 src = tasks.pop(task)
850 cancelled.append(task)
851 elif task.exception():
852 report.failed(src, str(task.exception()))
854 changed = Changed.YES if task.result() else Changed.NO
855 # If the file was written back or was successfully checked as
856 # well-formatted, store this information in the cache.
857 if write_back is WriteBack.YES or (
858 write_back is WriteBack.CHECK and changed is Changed.NO
860 sources_to_cache.append(src)
861 report.done(src, changed)
863 if sys.version_info >= (3, 7):
864 await asyncio.gather(*cancelled, return_exceptions=True)
866 await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
868 write_cache(cache, sources_to_cache, mode)
871 def format_file_in_place(
875 write_back: WriteBack = WriteBack.NO,
876 lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy
878 """Format file under `src` path. Return True if changed.
880 If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
882 `mode` and `fast` options are passed to :func:`format_file_contents`.
884 if src.suffix == ".pyi":
885 mode = replace(mode, is_pyi=True)
886 elif src.suffix == ".ipynb":
887 mode = replace(mode, is_ipynb=True)
889 then = datetime.utcfromtimestamp(src.stat().st_mtime)
890 with open(src, "rb") as buf:
891 src_contents, encoding, newline = decode_bytes(buf.read())
893 dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
894 except NothingChanged:
896 except JSONDecodeError:
898 f"File '{src}' cannot be parsed as valid Jupyter notebook."
901 if write_back == WriteBack.YES:
902 with open(src, "w", encoding=encoding, newline=newline) as f:
903 f.write(dst_contents)
904 elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
905 now = datetime.utcnow()
906 src_name = f"{src}\t{then} +0000"
907 dst_name = f"{src}\t{now} +0000"
909 diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name)
911 diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
913 if write_back == WriteBack.COLOR_DIFF:
914 diff_contents = color_diff(diff_contents)
916 with lock or nullcontext():
917 f = io.TextIOWrapper(
923 f = wrap_stream_for_windows(f)
924 f.write(diff_contents)
930 def format_stdin_to_stdout(
933 content: Optional[str] = None,
934 write_back: WriteBack = WriteBack.NO,
937 """Format file on stdin. Return True if changed.
939 If content is None, it's read from sys.stdin.
941 If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
942 write a diff to stdout. The `mode` argument is passed to
943 :func:`format_file_contents`.
945 then = datetime.utcnow()
948 src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
950 src, encoding, newline = content, "utf-8", ""
954 dst = format_file_contents(src, fast=fast, mode=mode)
957 except NothingChanged:
961 f = io.TextIOWrapper(
962 sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
964 if write_back == WriteBack.YES:
965 # Make sure there's a newline after the content
966 if dst and dst[-1] != "\n":
969 elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
970 now = datetime.utcnow()
971 src_name = f"STDIN\t{then} +0000"
972 dst_name = f"STDOUT\t{now} +0000"
973 d = diff(src, dst, src_name, dst_name)
974 if write_back == WriteBack.COLOR_DIFF:
976 f = wrap_stream_for_windows(f)
981 def check_stability_and_equivalence(
982 src_contents: str, dst_contents: str, *, mode: Mode
984 """Perform stability and equivalence checks.
986 Raise AssertionError if source and destination contents are not
987 equivalent, or if a second pass of the formatter would format the
990 assert_equivalent(src_contents, dst_contents)
991 assert_stable(src_contents, dst_contents, mode=mode)
994 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
995 """Reformat contents of a file and return new contents.
997 If `fast` is False, additionally confirm that the reformatted code is
998 valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
999 `mode` is passed to :func:`format_str`.
1001 if not src_contents.strip():
1002 raise NothingChanged
1005 dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode)
1007 dst_contents = format_str(src_contents, mode=mode)
1008 if src_contents == dst_contents:
1009 raise NothingChanged
1011 if not fast and not mode.is_ipynb:
1012 # Jupyter notebooks will already have been checked above.
1013 check_stability_and_equivalence(src_contents, dst_contents, mode=mode)
1017 def validate_cell(src: str, mode: Mode) -> None:
1018 """Check that cell does not already contain TransformerManager transformations,
1019 or non-Python cell magics, which might cause tokenizer_rt to break because of
1022 If a cell contains ``!ls``, then it'll be transformed to
1023 ``get_ipython().system('ls')``. However, if the cell originally contained
1024 ``get_ipython().system('ls')``, then it would get transformed in the same way:
1026 >>> TransformerManager().transform_cell("get_ipython().system('ls')")
1027 "get_ipython().system('ls')\n"
1028 >>> TransformerManager().transform_cell("!ls")
1029 "get_ipython().system('ls')\n"
1031 Due to the impossibility of safely roundtripping in such situations, cells
1032 containing transformed magics will be ignored.
1034 if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):
1035 raise NothingChanged
1038 and src.split()[0][2:] not in PYTHON_CELL_MAGICS | mode.python_cell_magics
1040 raise NothingChanged
1043 def format_cell(src: str, *, fast: bool, mode: Mode) -> str:
1044 """Format code in given cell of Jupyter notebook.
1048 - if cell has trailing semicolon, remove it;
1049 - if cell has IPython magics, mask them;
1051 - reinstate IPython magics;
1052 - reinstate trailing semicolon (if originally present);
1053 - strip trailing newlines.
1055 Cells with syntax errors will not be processed, as they
1056 could potentially be automagics or multi-line magics, which
1057 are currently not supported.
1059 validate_cell(src, mode)
1060 src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon(
1064 masked_src, replacements = mask_cell(src_without_trailing_semicolon)
1066 raise NothingChanged from None
1067 masked_dst = format_str(masked_src, mode=mode)
1069 check_stability_and_equivalence(masked_src, masked_dst, mode=mode)
1070 dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements)
1071 dst = put_trailing_semicolon_back(
1072 dst_without_trailing_semicolon, has_trailing_semicolon
1074 dst = dst.rstrip("\n")
1076 raise NothingChanged from None
1080 def validate_metadata(nb: MutableMapping[str, Any]) -> None:
1081 """If notebook is marked as non-Python, don't format it.
1083 All notebook metadata fields are optional, see
1084 https://nbformat.readthedocs.io/en/latest/format_description.html. So
1085 if a notebook has empty metadata, we will try to parse it anyway.
1087 language = nb.get("metadata", {}).get("language_info", {}).get("name", None)
1088 if language is not None and language != "python":
1089 raise NothingChanged from None
1092 def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
1093 """Format Jupyter notebook.
1095 Operate cell-by-cell, only on code cells, only for Python notebooks.
1096 If the ``.ipynb`` originally had a trailing newline, it'll be preserved.
1098 trailing_newline = src_contents[-1] == "\n"
1100 nb = json.loads(src_contents)
1101 validate_metadata(nb)
1102 for cell in nb["cells"]:
1103 if cell.get("cell_type", None) == "code":
1105 src = "".join(cell["source"])
1106 dst = format_cell(src, fast=fast, mode=mode)
1107 except NothingChanged:
1110 cell["source"] = dst.splitlines(keepends=True)
1113 dst_contents = json.dumps(nb, indent=1, ensure_ascii=False)
1114 if trailing_newline:
1115 dst_contents = dst_contents + "\n"
1118 raise NothingChanged
1121 def format_str(src_contents: str, *, mode: Mode) -> str:
1122 """Reformat a string and return new contents.
1124 `mode` determines formatting options, such as how many characters per line are
1128 >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
1129 def f(arg: str = "") -> None:
1132 A more complex example:
1135 ... black.format_str(
1136 ... "def f(arg:str='')->None: hey",
1137 ... mode=black.Mode(
1138 ... target_versions={black.TargetVersion.PY36},
1140 ... string_normalization=False,
1151 dst_contents = _format_str_once(src_contents, mode=mode)
1152 # Forced second pass to work around optional trailing commas (becoming
1153 # forced trailing commas on pass 2) interacting differently with optional
1154 # parentheses. Admittedly ugly.
1155 if src_contents != dst_contents:
1156 return _format_str_once(dst_contents, mode=mode)
1160 def _format_str_once(src_contents: str, *, mode: Mode) -> str:
1161 src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
1163 future_imports = get_future_imports(src_node)
1164 if mode.target_versions:
1165 versions = mode.target_versions
1167 versions = detect_target_versions(src_node, future_imports=future_imports)
1169 normalize_fmt_off(src_node)
1170 lines = LineGenerator(mode=mode)
1171 elt = EmptyLineTracker(is_pyi=mode.is_pyi)
1172 empty_line = Line(mode=mode)
1174 split_line_features = {
1176 for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
1177 if supports_feature(versions, feature)
1179 for current_line in lines.visit(src_node):
1180 dst_contents.append(str(empty_line) * after)
1181 before, after = elt.maybe_empty_lines(current_line)
1182 dst_contents.append(str(empty_line) * before)
1183 for line in transform_line(
1184 current_line, mode=mode, features=split_line_features
1186 dst_contents.append(str(line))
1187 return "".join(dst_contents)
1190 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
1191 """Return a tuple of (decoded_contents, encoding, newline).
1193 `newline` is either CRLF or LF but `decoded_contents` is decoded with
1194 universal newlines (i.e. only contains LF).
1196 srcbuf = io.BytesIO(src)
1197 encoding, lines = tokenize.detect_encoding(srcbuf.readline)
1199 return "", encoding, "\n"
1201 newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
1203 with io.TextIOWrapper(srcbuf, encoding) as tiow:
1204 return tiow.read(), encoding, newline
1207 def get_features_used( # noqa: C901
1208 node: Node, *, future_imports: Optional[Set[str]] = None
1210 """Return a set of (relatively) new Python features used in this file.
1212 Currently looking for:
1214 - underscores in numeric literals;
1215 - trailing commas after * or ** in function signatures and calls;
1216 - positional only arguments in function signatures and lambdas;
1217 - assignment expression;
1218 - relaxed decorator syntax;
1219 - usage of __future__ flags (annotations);
1220 - print / exec statements;
1222 features: Set[Feature] = set()
1225 FUTURE_FLAG_TO_FEATURE[future_import]
1226 for future_import in future_imports
1227 if future_import in FUTURE_FLAG_TO_FEATURE
1230 for n in node.pre_order():
1231 if is_string_token(n):
1232 value_head = n.value[:2]
1233 if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
1234 features.add(Feature.F_STRINGS)
1236 elif n.type == token.NUMBER:
1237 assert isinstance(n, Leaf)
1239 features.add(Feature.NUMERIC_UNDERSCORES)
1241 elif n.type == token.SLASH:
1242 if n.parent and n.parent.type in {
1247 features.add(Feature.POS_ONLY_ARGUMENTS)
1249 elif n.type == token.COLONEQUAL:
1250 features.add(Feature.ASSIGNMENT_EXPRESSIONS)
1252 elif n.type == syms.decorator:
1253 if len(n.children) > 1 and not is_simple_decorator_expression(
1256 features.add(Feature.RELAXED_DECORATORS)
1259 n.type in {syms.typedargslist, syms.arglist}
1261 and n.children[-1].type == token.COMMA
1263 if n.type == syms.typedargslist:
1264 feature = Feature.TRAILING_COMMA_IN_DEF
1266 feature = Feature.TRAILING_COMMA_IN_CALL
1268 for ch in n.children:
1269 if ch.type in STARS:
1270 features.add(feature)
1272 if ch.type == syms.argument:
1273 for argch in ch.children:
1274 if argch.type in STARS:
1275 features.add(feature)
1278 n.type in {syms.return_stmt, syms.yield_expr}
1279 and len(n.children) >= 2
1280 and n.children[1].type == syms.testlist_star_expr
1281 and any(child.type == syms.star_expr for child in n.children[1].children)
1283 features.add(Feature.UNPACKING_ON_FLOW)
1286 n.type == syms.annassign
1287 and len(n.children) >= 4
1288 and n.children[3].type == syms.testlist_star_expr
1290 features.add(Feature.ANN_ASSIGN_EXTENDED_RHS)
1295 def detect_target_versions(
1296 node: Node, *, future_imports: Optional[Set[str]] = None
1297 ) -> Set[TargetVersion]:
1298 """Detect the version to target based on the nodes used."""
1299 features = get_features_used(node, future_imports=future_imports)
1301 version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
1305 def get_future_imports(node: Node) -> Set[str]:
1306 """Return a set of __future__ imports in the file."""
1307 imports: Set[str] = set()
1309 def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
1310 for child in children:
1311 if isinstance(child, Leaf):
1312 if child.type == token.NAME:
1315 elif child.type == syms.import_as_name:
1316 orig_name = child.children[0]
1317 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
1318 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
1319 yield orig_name.value
1321 elif child.type == syms.import_as_names:
1322 yield from get_imports_from_children(child.children)
1325 raise AssertionError("Invalid syntax parsing imports")
1327 for child in node.children:
1328 if child.type != syms.simple_stmt:
1331 first_child = child.children[0]
1332 if isinstance(first_child, Leaf):
1333 # Continue looking if we see a docstring; otherwise stop.
1335 len(child.children) == 2
1336 and first_child.type == token.STRING
1337 and child.children[1].type == token.NEWLINE
1343 elif first_child.type == syms.import_from:
1344 module_name = first_child.children[1]
1345 if not isinstance(module_name, Leaf) or module_name.value != "__future__":
1348 imports |= set(get_imports_from_children(first_child.children[3:]))
1355 def assert_equivalent(src: str, dst: str) -> None:
1356 """Raise AssertionError if `src` and `dst` aren't equivalent."""
1358 src_ast = parse_ast(src)
1359 except Exception as exc:
1360 raise AssertionError(
1361 "cannot use --safe with this file; failed to parse source file AST: "
1363 "This could be caused by running Black with an older Python version "
1364 "that does not support new syntax used in your source file."
1368 dst_ast = parse_ast(dst)
1369 except Exception as exc:
1370 log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
1371 raise AssertionError(
1372 f"INTERNAL ERROR: Black produced invalid code: {exc}. "
1373 "Please report a bug on https://github.com/psf/black/issues. "
1374 f"This invalid output might be helpful: {log}"
1377 src_ast_str = "\n".join(stringify_ast(src_ast))
1378 dst_ast_str = "\n".join(stringify_ast(dst_ast))
1379 if src_ast_str != dst_ast_str:
1380 log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1381 raise AssertionError(
1382 "INTERNAL ERROR: Black produced code that is not equivalent to the"
1383 " source. Please report a bug on "
1384 f"https://github.com/psf/black/issues. This diff might be helpful: {log}"
1388 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1389 """Raise AssertionError if `dst` reformats differently the second time."""
1390 # We shouldn't call format_str() here, because that formats the string
1391 # twice and may hide a bug where we bounce back and forth between two
1393 newdst = _format_str_once(dst, mode=mode)
1397 diff(src, dst, "source", "first pass"),
1398 diff(dst, newdst, "first pass", "second pass"),
1400 raise AssertionError(
1401 "INTERNAL ERROR: Black produced different code on the second pass of the"
1402 " formatter. Please report a bug on https://github.com/psf/black/issues."
1403 f" This diff might be helpful: {log}"
1408 def nullcontext() -> Iterator[None]:
1409 """Return an empty context manager.
1411 To be used like `nullcontext` in Python 3.7.
1416 def patch_click() -> None:
1417 """Make Click not crash on Python 3.6 with LANG=C.
1419 On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1420 default which restricts paths that it can access during the lifetime of the
1421 application. Click refuses to work in this scenario by raising a RuntimeError.
1423 In case of Black the likelihood that non-ASCII characters are going to be used in
1424 file paths is minimal since it's Python source code. Moreover, this crash was
1425 spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1428 from click import core
1429 from click import _unicodefun
1430 except ModuleNotFoundError:
1433 for module in (core, _unicodefun):
1434 if hasattr(module, "_verify_python3_env"):
1435 module._verify_python3_env = lambda: None # type: ignore
1436 if hasattr(module, "_verify_python_env"):
1437 module._verify_python_env = lambda: None # type: ignore
1440 def patched_main() -> None:
1441 maybe_install_uvloop()
1447 if __name__ == "__main__":