]> git.madduck.net Git - etc/vim.git/blob - src/black/__init__.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Isolate command line tests from user-level config (#2851)
[etc/vim.git] / src / black / __init__.py
1 import asyncio
2 from json.decoder import JSONDecodeError
3 import json
4 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
5 from contextlib import contextmanager
6 from datetime import datetime
7 from enum import Enum
8 import io
9 from multiprocessing import Manager, freeze_support
10 import os
11 from pathlib import Path
12 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
13 import re
14 import signal
15 import sys
16 import tokenize
17 import traceback
18 from typing import (
19     Any,
20     Dict,
21     Generator,
22     Iterator,
23     List,
24     MutableMapping,
25     Optional,
26     Pattern,
27     Sequence,
28     Set,
29     Sized,
30     Tuple,
31     Union,
32 )
33
34 import click
35 from click.core import ParameterSource
36 from dataclasses import replace
37 from mypy_extensions import mypyc_attr
38
39 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
40 from black.const import STDIN_PLACEHOLDER
41 from black.nodes import STARS, syms, is_simple_decorator_expression
42 from black.nodes import is_string_token
43 from black.lines import Line, EmptyLineTracker
44 from black.linegen import transform_line, LineGenerator, LN
45 from black.comments import normalize_fmt_off
46 from black.mode import FUTURE_FLAG_TO_FEATURE, Mode, TargetVersion
47 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
48 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
49 from black.concurrency import cancel, shutdown, maybe_install_uvloop
50 from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err
51 from black.report import Report, Changed, NothingChanged
52 from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml
53 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
54 from black.files import wrap_stream_for_windows
55 from black.parsing import InvalidInput  # noqa F401
56 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
57 from black.handle_ipynb_magics import (
58     mask_cell,
59     unmask_cell,
60     remove_trailing_semicolon,
61     put_trailing_semicolon_back,
62     TRANSFORMED_MAGICS,
63     PYTHON_CELL_MAGICS,
64     jupyter_dependencies_are_installed,
65 )
66
67
68 # lib2to3 fork
69 from blib2to3.pytree import Node, Leaf
70 from blib2to3.pgen2 import token
71
72 from _black_version import version as __version__
73
74 COMPILED = Path(__file__).suffix in (".pyd", ".so")
75
76 # types
77 FileContent = str
78 Encoding = str
79 NewLine = str
80
81
82 class WriteBack(Enum):
83     NO = 0
84     YES = 1
85     DIFF = 2
86     CHECK = 3
87     COLOR_DIFF = 4
88
89     @classmethod
90     def from_configuration(
91         cls, *, check: bool, diff: bool, color: bool = False
92     ) -> "WriteBack":
93         if check and not diff:
94             return cls.CHECK
95
96         if diff and color:
97             return cls.COLOR_DIFF
98
99         return cls.DIFF if diff else cls.YES
100
101
102 # Legacy name, left for integrations.
103 FileMode = Mode
104
105 DEFAULT_WORKERS = os.cpu_count()
106
107
108 def read_pyproject_toml(
109     ctx: click.Context, param: click.Parameter, value: Optional[str]
110 ) -> Optional[str]:
111     """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
112
113     Returns the path to a successfully found and read configuration file, None
114     otherwise.
115     """
116     if not value:
117         value = find_pyproject_toml(ctx.params.get("src", ()))
118         if value is None:
119             return None
120
121     try:
122         config = parse_pyproject_toml(value)
123     except (OSError, ValueError) as e:
124         raise click.FileError(
125             filename=value, hint=f"Error reading configuration file: {e}"
126         ) from None
127
128     if not config:
129         return None
130     else:
131         # Sanitize the values to be Click friendly. For more information please see:
132         # https://github.com/psf/black/issues/1458
133         # https://github.com/pallets/click/issues/1567
134         config = {
135             k: str(v) if not isinstance(v, (list, dict)) else v
136             for k, v in config.items()
137         }
138
139     target_version = config.get("target_version")
140     if target_version is not None and not isinstance(target_version, list):
141         raise click.BadOptionUsage(
142             "target-version", "Config key target-version must be a list"
143         )
144
145     default_map: Dict[str, Any] = {}
146     if ctx.default_map:
147         default_map.update(ctx.default_map)
148     default_map.update(config)
149
150     ctx.default_map = default_map
151     return value
152
153
154 def target_version_option_callback(
155     c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
156 ) -> List[TargetVersion]:
157     """Compute the target versions from a --target-version flag.
158
159     This is its own function because mypy couldn't infer the type correctly
160     when it was a lambda, causing mypyc trouble.
161     """
162     return [TargetVersion[val.upper()] for val in v]
163
164
165 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
166     """Compile a regular expression string in `regex`.
167
168     If it contains newlines, use verbose mode.
169     """
170     if "\n" in regex:
171         regex = "(?x)" + regex
172     compiled: Pattern[str] = re.compile(regex)
173     return compiled
174
175
176 def validate_regex(
177     ctx: click.Context,
178     param: click.Parameter,
179     value: Optional[str],
180 ) -> Optional[Pattern[str]]:
181     try:
182         return re_compile_maybe_verbose(value) if value is not None else None
183     except re.error as e:
184         raise click.BadParameter(f"Not a valid regular expression: {e}") from None
185
186
187 @click.command(
188     context_settings={"help_option_names": ["-h", "--help"]},
189     # While Click does set this field automatically using the docstring, mypyc
190     # (annoyingly) strips 'em so we need to set it here too.
191     help="The uncompromising code formatter.",
192 )
193 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
194 @click.option(
195     "-l",
196     "--line-length",
197     type=int,
198     default=DEFAULT_LINE_LENGTH,
199     help="How many characters per line to allow.",
200     show_default=True,
201 )
202 @click.option(
203     "-t",
204     "--target-version",
205     type=click.Choice([v.name.lower() for v in TargetVersion]),
206     callback=target_version_option_callback,
207     multiple=True,
208     help=(
209         "Python versions that should be supported by Black's output. [default: per-file"
210         " auto-detection]"
211     ),
212 )
213 @click.option(
214     "--pyi",
215     is_flag=True,
216     help=(
217         "Format all input files like typing stubs regardless of file extension (useful"
218         " when piping source on standard input)."
219     ),
220 )
221 @click.option(
222     "--ipynb",
223     is_flag=True,
224     help=(
225         "Format all input files like Jupyter Notebooks regardless of file extension "
226         "(useful when piping source on standard input)."
227     ),
228 )
229 @click.option(
230     "--python-cell-magics",
231     multiple=True,
232     help=(
233         "When processing Jupyter Notebooks, add the given magic to the list"
234         f" of known python-magics ({', '.join(PYTHON_CELL_MAGICS)})."
235         " Useful for formatting cells with custom python magics."
236     ),
237     default=[],
238 )
239 @click.option(
240     "-S",
241     "--skip-string-normalization",
242     is_flag=True,
243     help="Don't normalize string quotes or prefixes.",
244 )
245 @click.option(
246     "-C",
247     "--skip-magic-trailing-comma",
248     is_flag=True,
249     help="Don't use trailing commas as a reason to split lines.",
250 )
251 @click.option(
252     "--experimental-string-processing",
253     is_flag=True,
254     hidden=True,
255     help="(DEPRECATED and now included in --preview) Normalize string literals.",
256 )
257 @click.option(
258     "--preview",
259     is_flag=True,
260     help=(
261         "Enable potentially disruptive style changes that may be added to Black's main"
262         " functionality in the next major release."
263     ),
264 )
265 @click.option(
266     "--check",
267     is_flag=True,
268     help=(
269         "Don't write the files back, just return the status. Return code 0 means"
270         " nothing would change. Return code 1 means some files would be reformatted."
271         " Return code 123 means there was an internal error."
272     ),
273 )
274 @click.option(
275     "--diff",
276     is_flag=True,
277     help="Don't write the files back, just output a diff for each file on stdout.",
278 )
279 @click.option(
280     "--color/--no-color",
281     is_flag=True,
282     help="Show colored diff. Only applies when `--diff` is given.",
283 )
284 @click.option(
285     "--fast/--safe",
286     is_flag=True,
287     help="If --fast given, skip temporary sanity checks. [default: --safe]",
288 )
289 @click.option(
290     "--required-version",
291     type=str,
292     help=(
293         "Require a specific version of Black to be running (useful for unifying results"
294         " across many environments e.g. with a pyproject.toml file). It can be"
295         " either a major version number or an exact version."
296     ),
297 )
298 @click.option(
299     "--include",
300     type=str,
301     default=DEFAULT_INCLUDES,
302     callback=validate_regex,
303     help=(
304         "A regular expression that matches files and directories that should be"
305         " included on recursive searches. An empty value means all files are included"
306         " regardless of the name. Use forward slashes for directories on all platforms"
307         " (Windows, too). Exclusions are calculated first, inclusions later."
308     ),
309     show_default=True,
310 )
311 @click.option(
312     "--exclude",
313     type=str,
314     callback=validate_regex,
315     help=(
316         "A regular expression that matches files and directories that should be"
317         " excluded on recursive searches. An empty value means no paths are excluded."
318         " Use forward slashes for directories on all platforms (Windows, too)."
319         " Exclusions are calculated first, inclusions later. [default:"
320         f" {DEFAULT_EXCLUDES}]"
321     ),
322     show_default=False,
323 )
324 @click.option(
325     "--extend-exclude",
326     type=str,
327     callback=validate_regex,
328     help=(
329         "Like --exclude, but adds additional files and directories on top of the"
330         " excluded ones. (Useful if you simply want to add to the default)"
331     ),
332 )
333 @click.option(
334     "--force-exclude",
335     type=str,
336     callback=validate_regex,
337     help=(
338         "Like --exclude, but files and directories matching this regex will be "
339         "excluded even when they are passed explicitly as arguments."
340     ),
341 )
342 @click.option(
343     "--stdin-filename",
344     type=str,
345     help=(
346         "The name of the file when passing it through stdin. Useful to make "
347         "sure Black will respect --force-exclude option on some "
348         "editors that rely on using stdin."
349     ),
350 )
351 @click.option(
352     "-W",
353     "--workers",
354     type=click.IntRange(min=1),
355     default=DEFAULT_WORKERS,
356     show_default=True,
357     help="Number of parallel workers",
358 )
359 @click.option(
360     "-q",
361     "--quiet",
362     is_flag=True,
363     help=(
364         "Don't emit non-error messages to stderr. Errors are still emitted; silence"
365         " those with 2>/dev/null."
366     ),
367 )
368 @click.option(
369     "-v",
370     "--verbose",
371     is_flag=True,
372     help=(
373         "Also emit messages to stderr about files that were not changed or were ignored"
374         " due to exclusion patterns."
375     ),
376 )
377 @click.version_option(
378     version=__version__,
379     message=f"%(prog)s, %(version)s (compiled: {'yes' if COMPILED else 'no'})",
380 )
381 @click.argument(
382     "src",
383     nargs=-1,
384     type=click.Path(
385         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
386     ),
387     is_eager=True,
388     metavar="SRC ...",
389 )
390 @click.option(
391     "--config",
392     type=click.Path(
393         exists=True,
394         file_okay=True,
395         dir_okay=False,
396         readable=True,
397         allow_dash=False,
398         path_type=str,
399     ),
400     is_eager=True,
401     callback=read_pyproject_toml,
402     help="Read configuration from FILE path.",
403 )
404 @click.pass_context
405 def main(
406     ctx: click.Context,
407     code: Optional[str],
408     line_length: int,
409     target_version: List[TargetVersion],
410     check: bool,
411     diff: bool,
412     color: bool,
413     fast: bool,
414     pyi: bool,
415     ipynb: bool,
416     python_cell_magics: Sequence[str],
417     skip_string_normalization: bool,
418     skip_magic_trailing_comma: bool,
419     experimental_string_processing: bool,
420     preview: bool,
421     quiet: bool,
422     verbose: bool,
423     required_version: Optional[str],
424     include: Pattern[str],
425     exclude: Optional[Pattern[str]],
426     extend_exclude: Optional[Pattern[str]],
427     force_exclude: Optional[Pattern[str]],
428     stdin_filename: Optional[str],
429     workers: int,
430     src: Tuple[str, ...],
431     config: Optional[str],
432 ) -> None:
433     """The uncompromising code formatter."""
434     ctx.ensure_object(dict)
435
436     if src and code is not None:
437         out(
438             main.get_usage(ctx)
439             + "\n\n'SRC' and 'code' cannot be passed simultaneously."
440         )
441         ctx.exit(1)
442     if not src and code is None:
443         out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.")
444         ctx.exit(1)
445
446     root, method = find_project_root(src) if code is None else (None, None)
447     ctx.obj["root"] = root
448
449     if verbose:
450         if root:
451             out(
452                 f"Identified `{root}` as project root containing a {method}.",
453                 fg="blue",
454             )
455
456             normalized = [
457                 (normalize_path_maybe_ignore(Path(source), root), source)
458                 for source in src
459             ]
460             srcs_string = ", ".join(
461                 [
462                     f'"{_norm}"'
463                     if _norm
464                     else f'\033[31m"{source} (skipping - invalid)"\033[34m'
465                     for _norm, source in normalized
466                 ]
467             )
468             out(f"Sources to be formatted: {srcs_string}", fg="blue")
469
470         if config:
471             config_source = ctx.get_parameter_source("config")
472             if config_source in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP):
473                 out("Using configuration from project root.", fg="blue")
474             else:
475                 out(f"Using configuration in '{config}'.", fg="blue")
476
477     error_msg = "Oh no! 💥 💔 💥"
478     if (
479         required_version
480         and required_version != __version__
481         and required_version != __version__.split(".")[0]
482     ):
483         err(
484             f"{error_msg} The required version `{required_version}` does not match"
485             f" the running version `{__version__}`!"
486         )
487         ctx.exit(1)
488     if ipynb and pyi:
489         err("Cannot pass both `pyi` and `ipynb` flags!")
490         ctx.exit(1)
491
492     write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
493     if target_version:
494         versions = set(target_version)
495     else:
496         # We'll autodetect later.
497         versions = set()
498     mode = Mode(
499         target_versions=versions,
500         line_length=line_length,
501         is_pyi=pyi,
502         is_ipynb=ipynb,
503         string_normalization=not skip_string_normalization,
504         magic_trailing_comma=not skip_magic_trailing_comma,
505         experimental_string_processing=experimental_string_processing,
506         preview=preview,
507         python_cell_magics=set(python_cell_magics),
508     )
509
510     if code is not None:
511         # Run in quiet mode by default with -c; the extra output isn't useful.
512         # You can still pass -v to get verbose output.
513         quiet = True
514
515     report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
516
517     if code is not None:
518         reformat_code(
519             content=code, fast=fast, write_back=write_back, mode=mode, report=report
520         )
521     else:
522         try:
523             sources = get_sources(
524                 ctx=ctx,
525                 src=src,
526                 quiet=quiet,
527                 verbose=verbose,
528                 include=include,
529                 exclude=exclude,
530                 extend_exclude=extend_exclude,
531                 force_exclude=force_exclude,
532                 report=report,
533                 stdin_filename=stdin_filename,
534             )
535         except GitWildMatchPatternError:
536             ctx.exit(1)
537
538         path_empty(
539             sources,
540             "No Python files are present to be formatted. Nothing to do 😴",
541             quiet,
542             verbose,
543             ctx,
544         )
545
546         if len(sources) == 1:
547             reformat_one(
548                 src=sources.pop(),
549                 fast=fast,
550                 write_back=write_back,
551                 mode=mode,
552                 report=report,
553             )
554         else:
555             reformat_many(
556                 sources=sources,
557                 fast=fast,
558                 write_back=write_back,
559                 mode=mode,
560                 report=report,
561                 workers=workers,
562             )
563
564     if verbose or not quiet:
565         if code is None and (verbose or report.change_count or report.failure_count):
566             out()
567         out(error_msg if report.return_code else "All done! ✨ 🍰 ✨")
568         if code is None:
569             click.echo(str(report), err=True)
570     ctx.exit(report.return_code)
571
572
573 def get_sources(
574     *,
575     ctx: click.Context,
576     src: Tuple[str, ...],
577     quiet: bool,
578     verbose: bool,
579     include: Pattern[str],
580     exclude: Optional[Pattern[str]],
581     extend_exclude: Optional[Pattern[str]],
582     force_exclude: Optional[Pattern[str]],
583     report: "Report",
584     stdin_filename: Optional[str],
585 ) -> Set[Path]:
586     """Compute the set of files to be formatted."""
587     sources: Set[Path] = set()
588
589     if exclude is None:
590         exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
591         gitignore = get_gitignore(ctx.obj["root"])
592     else:
593         gitignore = None
594
595     for s in src:
596         if s == "-" and stdin_filename:
597             p = Path(stdin_filename)
598             is_stdin = True
599         else:
600             p = Path(s)
601             is_stdin = False
602
603         if is_stdin or p.is_file():
604             normalized_path = normalize_path_maybe_ignore(p, ctx.obj["root"], report)
605             if normalized_path is None:
606                 continue
607
608             normalized_path = "/" + normalized_path
609             # Hard-exclude any files that matches the `--force-exclude` regex.
610             if force_exclude:
611                 force_exclude_match = force_exclude.search(normalized_path)
612             else:
613                 force_exclude_match = None
614             if force_exclude_match and force_exclude_match.group(0):
615                 report.path_ignored(p, "matches the --force-exclude regular expression")
616                 continue
617
618             if is_stdin:
619                 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
620
621             if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
622                 verbose=verbose, quiet=quiet
623             ):
624                 continue
625
626             sources.add(p)
627         elif p.is_dir():
628             sources.update(
629                 gen_python_files(
630                     p.iterdir(),
631                     ctx.obj["root"],
632                     include,
633                     exclude,
634                     extend_exclude,
635                     force_exclude,
636                     report,
637                     gitignore,
638                     verbose=verbose,
639                     quiet=quiet,
640                 )
641             )
642         elif s == "-":
643             sources.add(p)
644         else:
645             err(f"invalid path: {s}")
646     return sources
647
648
649 def path_empty(
650     src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
651 ) -> None:
652     """
653     Exit if there is no `src` provided for formatting
654     """
655     if not src:
656         if verbose or not quiet:
657             out(msg)
658         ctx.exit(0)
659
660
661 def reformat_code(
662     content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report
663 ) -> None:
664     """
665     Reformat and print out `content` without spawning child processes.
666     Similar to `reformat_one`, but for string content.
667
668     `fast`, `write_back`, and `mode` options are passed to
669     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
670     """
671     path = Path("<string>")
672     try:
673         changed = Changed.NO
674         if format_stdin_to_stdout(
675             content=content, fast=fast, write_back=write_back, mode=mode
676         ):
677             changed = Changed.YES
678         report.done(path, changed)
679     except Exception as exc:
680         if report.verbose:
681             traceback.print_exc()
682         report.failed(path, str(exc))
683
684
685 def reformat_one(
686     src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
687 ) -> None:
688     """Reformat a single file under `src` without spawning child processes.
689
690     `fast`, `write_back`, and `mode` options are passed to
691     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
692     """
693     try:
694         changed = Changed.NO
695
696         if str(src) == "-":
697             is_stdin = True
698         elif str(src).startswith(STDIN_PLACEHOLDER):
699             is_stdin = True
700             # Use the original name again in case we want to print something
701             # to the user
702             src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
703         else:
704             is_stdin = False
705
706         if is_stdin:
707             if src.suffix == ".pyi":
708                 mode = replace(mode, is_pyi=True)
709             elif src.suffix == ".ipynb":
710                 mode = replace(mode, is_ipynb=True)
711             if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
712                 changed = Changed.YES
713         else:
714             cache: Cache = {}
715             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
716                 cache = read_cache(mode)
717                 res_src = src.resolve()
718                 res_src_s = str(res_src)
719                 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
720                     changed = Changed.CACHED
721             if changed is not Changed.CACHED and format_file_in_place(
722                 src, fast=fast, write_back=write_back, mode=mode
723             ):
724                 changed = Changed.YES
725             if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
726                 write_back is WriteBack.CHECK and changed is Changed.NO
727             ):
728                 write_cache(cache, [src], mode)
729         report.done(src, changed)
730     except Exception as exc:
731         if report.verbose:
732             traceback.print_exc()
733         report.failed(src, str(exc))
734
735
736 # diff-shades depends on being to monkeypatch this function to operate. I know it's
737 # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26
738 @mypyc_attr(patchable=True)
739 def reformat_many(
740     sources: Set[Path],
741     fast: bool,
742     write_back: WriteBack,
743     mode: Mode,
744     report: "Report",
745     workers: Optional[int],
746 ) -> None:
747     """Reformat multiple files using a ProcessPoolExecutor."""
748     executor: Executor
749     loop = asyncio.get_event_loop()
750     worker_count = workers if workers is not None else DEFAULT_WORKERS
751     if sys.platform == "win32":
752         # Work around https://bugs.python.org/issue26903
753         assert worker_count is not None
754         worker_count = min(worker_count, 60)
755     try:
756         executor = ProcessPoolExecutor(max_workers=worker_count)
757     except (ImportError, NotImplementedError, OSError):
758         # we arrive here if the underlying system does not support multi-processing
759         # like in AWS Lambda or Termux, in which case we gracefully fallback to
760         # a ThreadPoolExecutor with just a single worker (more workers would not do us
761         # any good due to the Global Interpreter Lock)
762         executor = ThreadPoolExecutor(max_workers=1)
763
764     try:
765         loop.run_until_complete(
766             schedule_formatting(
767                 sources=sources,
768                 fast=fast,
769                 write_back=write_back,
770                 mode=mode,
771                 report=report,
772                 loop=loop,
773                 executor=executor,
774             )
775         )
776     finally:
777         shutdown(loop)
778         if executor is not None:
779             executor.shutdown()
780
781
782 async def schedule_formatting(
783     sources: Set[Path],
784     fast: bool,
785     write_back: WriteBack,
786     mode: Mode,
787     report: "Report",
788     loop: asyncio.AbstractEventLoop,
789     executor: Executor,
790 ) -> None:
791     """Run formatting of `sources` in parallel using the provided `executor`.
792
793     (Use ProcessPoolExecutors for actual parallelism.)
794
795     `write_back`, `fast`, and `mode` options are passed to
796     :func:`format_file_in_place`.
797     """
798     cache: Cache = {}
799     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
800         cache = read_cache(mode)
801         sources, cached = filter_cached(cache, sources)
802         for src in sorted(cached):
803             report.done(src, Changed.CACHED)
804     if not sources:
805         return
806
807     cancelled = []
808     sources_to_cache = []
809     lock = None
810     if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
811         # For diff output, we need locks to ensure we don't interleave output
812         # from different processes.
813         manager = Manager()
814         lock = manager.Lock()
815     tasks = {
816         asyncio.ensure_future(
817             loop.run_in_executor(
818                 executor, format_file_in_place, src, fast, mode, write_back, lock
819             )
820         ): src
821         for src in sorted(sources)
822     }
823     pending = tasks.keys()
824     try:
825         loop.add_signal_handler(signal.SIGINT, cancel, pending)
826         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
827     except NotImplementedError:
828         # There are no good alternatives for these on Windows.
829         pass
830     while pending:
831         done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
832         for task in done:
833             src = tasks.pop(task)
834             if task.cancelled():
835                 cancelled.append(task)
836             elif task.exception():
837                 report.failed(src, str(task.exception()))
838             else:
839                 changed = Changed.YES if task.result() else Changed.NO
840                 # If the file was written back or was successfully checked as
841                 # well-formatted, store this information in the cache.
842                 if write_back is WriteBack.YES or (
843                     write_back is WriteBack.CHECK and changed is Changed.NO
844                 ):
845                     sources_to_cache.append(src)
846                 report.done(src, changed)
847     if cancelled:
848         if sys.version_info >= (3, 7):
849             await asyncio.gather(*cancelled, return_exceptions=True)
850         else:
851             await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
852     if sources_to_cache:
853         write_cache(cache, sources_to_cache, mode)
854
855
856 def format_file_in_place(
857     src: Path,
858     fast: bool,
859     mode: Mode,
860     write_back: WriteBack = WriteBack.NO,
861     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
862 ) -> bool:
863     """Format file under `src` path. Return True if changed.
864
865     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
866     code to the file.
867     `mode` and `fast` options are passed to :func:`format_file_contents`.
868     """
869     if src.suffix == ".pyi":
870         mode = replace(mode, is_pyi=True)
871     elif src.suffix == ".ipynb":
872         mode = replace(mode, is_ipynb=True)
873
874     then = datetime.utcfromtimestamp(src.stat().st_mtime)
875     with open(src, "rb") as buf:
876         src_contents, encoding, newline = decode_bytes(buf.read())
877     try:
878         dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
879     except NothingChanged:
880         return False
881     except JSONDecodeError:
882         raise ValueError(
883             f"File '{src}' cannot be parsed as valid Jupyter notebook."
884         ) from None
885
886     if write_back == WriteBack.YES:
887         with open(src, "w", encoding=encoding, newline=newline) as f:
888             f.write(dst_contents)
889     elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
890         now = datetime.utcnow()
891         src_name = f"{src}\t{then} +0000"
892         dst_name = f"{src}\t{now} +0000"
893         if mode.is_ipynb:
894             diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name)
895         else:
896             diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
897
898         if write_back == WriteBack.COLOR_DIFF:
899             diff_contents = color_diff(diff_contents)
900
901         with lock or nullcontext():
902             f = io.TextIOWrapper(
903                 sys.stdout.buffer,
904                 encoding=encoding,
905                 newline=newline,
906                 write_through=True,
907             )
908             f = wrap_stream_for_windows(f)
909             f.write(diff_contents)
910             f.detach()
911
912     return True
913
914
915 def format_stdin_to_stdout(
916     fast: bool,
917     *,
918     content: Optional[str] = None,
919     write_back: WriteBack = WriteBack.NO,
920     mode: Mode,
921 ) -> bool:
922     """Format file on stdin. Return True if changed.
923
924     If content is None, it's read from sys.stdin.
925
926     If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
927     write a diff to stdout. The `mode` argument is passed to
928     :func:`format_file_contents`.
929     """
930     then = datetime.utcnow()
931
932     if content is None:
933         src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
934     else:
935         src, encoding, newline = content, "utf-8", ""
936
937     dst = src
938     try:
939         dst = format_file_contents(src, fast=fast, mode=mode)
940         return True
941
942     except NothingChanged:
943         return False
944
945     finally:
946         f = io.TextIOWrapper(
947             sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
948         )
949         if write_back == WriteBack.YES:
950             # Make sure there's a newline after the content
951             if dst and dst[-1] != "\n":
952                 dst += "\n"
953             f.write(dst)
954         elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
955             now = datetime.utcnow()
956             src_name = f"STDIN\t{then} +0000"
957             dst_name = f"STDOUT\t{now} +0000"
958             d = diff(src, dst, src_name, dst_name)
959             if write_back == WriteBack.COLOR_DIFF:
960                 d = color_diff(d)
961                 f = wrap_stream_for_windows(f)
962             f.write(d)
963         f.detach()
964
965
966 def check_stability_and_equivalence(
967     src_contents: str, dst_contents: str, *, mode: Mode
968 ) -> None:
969     """Perform stability and equivalence checks.
970
971     Raise AssertionError if source and destination contents are not
972     equivalent, or if a second pass of the formatter would format the
973     content differently.
974     """
975     assert_equivalent(src_contents, dst_contents)
976     assert_stable(src_contents, dst_contents, mode=mode)
977
978
979 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
980     """Reformat contents of a file and return new contents.
981
982     If `fast` is False, additionally confirm that the reformatted code is
983     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
984     `mode` is passed to :func:`format_str`.
985     """
986     if not src_contents.strip():
987         raise NothingChanged
988
989     if mode.is_ipynb:
990         dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode)
991     else:
992         dst_contents = format_str(src_contents, mode=mode)
993     if src_contents == dst_contents:
994         raise NothingChanged
995
996     if not fast and not mode.is_ipynb:
997         # Jupyter notebooks will already have been checked above.
998         check_stability_and_equivalence(src_contents, dst_contents, mode=mode)
999     return dst_contents
1000
1001
1002 def validate_cell(src: str, mode: Mode) -> None:
1003     """Check that cell does not already contain TransformerManager transformations,
1004     or non-Python cell magics, which might cause tokenizer_rt to break because of
1005     indentations.
1006
1007     If a cell contains ``!ls``, then it'll be transformed to
1008     ``get_ipython().system('ls')``. However, if the cell originally contained
1009     ``get_ipython().system('ls')``, then it would get transformed in the same way:
1010
1011         >>> TransformerManager().transform_cell("get_ipython().system('ls')")
1012         "get_ipython().system('ls')\n"
1013         >>> TransformerManager().transform_cell("!ls")
1014         "get_ipython().system('ls')\n"
1015
1016     Due to the impossibility of safely roundtripping in such situations, cells
1017     containing transformed magics will be ignored.
1018     """
1019     if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):
1020         raise NothingChanged
1021     if (
1022         src[:2] == "%%"
1023         and src.split()[0][2:] not in PYTHON_CELL_MAGICS | mode.python_cell_magics
1024     ):
1025         raise NothingChanged
1026
1027
1028 def format_cell(src: str, *, fast: bool, mode: Mode) -> str:
1029     """Format code in given cell of Jupyter notebook.
1030
1031     General idea is:
1032
1033       - if cell has trailing semicolon, remove it;
1034       - if cell has IPython magics, mask them;
1035       - format cell;
1036       - reinstate IPython magics;
1037       - reinstate trailing semicolon (if originally present);
1038       - strip trailing newlines.
1039
1040     Cells with syntax errors will not be processed, as they
1041     could potentially be automagics or multi-line magics, which
1042     are currently not supported.
1043     """
1044     validate_cell(src, mode)
1045     src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon(
1046         src
1047     )
1048     try:
1049         masked_src, replacements = mask_cell(src_without_trailing_semicolon)
1050     except SyntaxError:
1051         raise NothingChanged from None
1052     masked_dst = format_str(masked_src, mode=mode)
1053     if not fast:
1054         check_stability_and_equivalence(masked_src, masked_dst, mode=mode)
1055     dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements)
1056     dst = put_trailing_semicolon_back(
1057         dst_without_trailing_semicolon, has_trailing_semicolon
1058     )
1059     dst = dst.rstrip("\n")
1060     if dst == src:
1061         raise NothingChanged from None
1062     return dst
1063
1064
1065 def validate_metadata(nb: MutableMapping[str, Any]) -> None:
1066     """If notebook is marked as non-Python, don't format it.
1067
1068     All notebook metadata fields are optional, see
1069     https://nbformat.readthedocs.io/en/latest/format_description.html. So
1070     if a notebook has empty metadata, we will try to parse it anyway.
1071     """
1072     language = nb.get("metadata", {}).get("language_info", {}).get("name", None)
1073     if language is not None and language != "python":
1074         raise NothingChanged from None
1075
1076
1077 def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
1078     """Format Jupyter notebook.
1079
1080     Operate cell-by-cell, only on code cells, only for Python notebooks.
1081     If the ``.ipynb`` originally had a trailing newline, it'll be preserved.
1082     """
1083     trailing_newline = src_contents[-1] == "\n"
1084     modified = False
1085     nb = json.loads(src_contents)
1086     validate_metadata(nb)
1087     for cell in nb["cells"]:
1088         if cell.get("cell_type", None) == "code":
1089             try:
1090                 src = "".join(cell["source"])
1091                 dst = format_cell(src, fast=fast, mode=mode)
1092             except NothingChanged:
1093                 pass
1094             else:
1095                 cell["source"] = dst.splitlines(keepends=True)
1096                 modified = True
1097     if modified:
1098         dst_contents = json.dumps(nb, indent=1, ensure_ascii=False)
1099         if trailing_newline:
1100             dst_contents = dst_contents + "\n"
1101         return dst_contents
1102     else:
1103         raise NothingChanged
1104
1105
1106 def format_str(src_contents: str, *, mode: Mode) -> str:
1107     """Reformat a string and return new contents.
1108
1109     `mode` determines formatting options, such as how many characters per line are
1110     allowed.  Example:
1111
1112     >>> import black
1113     >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
1114     def f(arg: str = "") -> None:
1115         ...
1116
1117     A more complex example:
1118
1119     >>> print(
1120     ...   black.format_str(
1121     ...     "def f(arg:str='')->None: hey",
1122     ...     mode=black.Mode(
1123     ...       target_versions={black.TargetVersion.PY36},
1124     ...       line_length=10,
1125     ...       string_normalization=False,
1126     ...       is_pyi=False,
1127     ...     ),
1128     ...   ),
1129     ... )
1130     def f(
1131         arg: str = '',
1132     ) -> None:
1133         hey
1134
1135     """
1136     dst_contents = _format_str_once(src_contents, mode=mode)
1137     # Forced second pass to work around optional trailing commas (becoming
1138     # forced trailing commas on pass 2) interacting differently with optional
1139     # parentheses.  Admittedly ugly.
1140     if src_contents != dst_contents:
1141         return _format_str_once(dst_contents, mode=mode)
1142     return dst_contents
1143
1144
1145 def _format_str_once(src_contents: str, *, mode: Mode) -> str:
1146     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
1147     dst_contents = []
1148     future_imports = get_future_imports(src_node)
1149     if mode.target_versions:
1150         versions = mode.target_versions
1151     else:
1152         versions = detect_target_versions(src_node, future_imports=future_imports)
1153
1154     normalize_fmt_off(src_node)
1155     lines = LineGenerator(mode=mode)
1156     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
1157     empty_line = Line(mode=mode)
1158     after = 0
1159     split_line_features = {
1160         feature
1161         for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
1162         if supports_feature(versions, feature)
1163     }
1164     for current_line in lines.visit(src_node):
1165         dst_contents.append(str(empty_line) * after)
1166         before, after = elt.maybe_empty_lines(current_line)
1167         dst_contents.append(str(empty_line) * before)
1168         for line in transform_line(
1169             current_line, mode=mode, features=split_line_features
1170         ):
1171             dst_contents.append(str(line))
1172     return "".join(dst_contents)
1173
1174
1175 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
1176     """Return a tuple of (decoded_contents, encoding, newline).
1177
1178     `newline` is either CRLF or LF but `decoded_contents` is decoded with
1179     universal newlines (i.e. only contains LF).
1180     """
1181     srcbuf = io.BytesIO(src)
1182     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
1183     if not lines:
1184         return "", encoding, "\n"
1185
1186     newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
1187     srcbuf.seek(0)
1188     with io.TextIOWrapper(srcbuf, encoding) as tiow:
1189         return tiow.read(), encoding, newline
1190
1191
1192 def get_features_used(  # noqa: C901
1193     node: Node, *, future_imports: Optional[Set[str]] = None
1194 ) -> Set[Feature]:
1195     """Return a set of (relatively) new Python features used in this file.
1196
1197     Currently looking for:
1198     - f-strings;
1199     - underscores in numeric literals;
1200     - trailing commas after * or ** in function signatures and calls;
1201     - positional only arguments in function signatures and lambdas;
1202     - assignment expression;
1203     - relaxed decorator syntax;
1204     - usage of __future__ flags (annotations);
1205     - print / exec statements;
1206     """
1207     features: Set[Feature] = set()
1208     if future_imports:
1209         features |= {
1210             FUTURE_FLAG_TO_FEATURE[future_import]
1211             for future_import in future_imports
1212             if future_import in FUTURE_FLAG_TO_FEATURE
1213         }
1214
1215     for n in node.pre_order():
1216         if is_string_token(n):
1217             value_head = n.value[:2]
1218             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
1219                 features.add(Feature.F_STRINGS)
1220
1221         elif n.type == token.NUMBER:
1222             assert isinstance(n, Leaf)
1223             if "_" in n.value:
1224                 features.add(Feature.NUMERIC_UNDERSCORES)
1225
1226         elif n.type == token.SLASH:
1227             if n.parent and n.parent.type in {
1228                 syms.typedargslist,
1229                 syms.arglist,
1230                 syms.varargslist,
1231             }:
1232                 features.add(Feature.POS_ONLY_ARGUMENTS)
1233
1234         elif n.type == token.COLONEQUAL:
1235             features.add(Feature.ASSIGNMENT_EXPRESSIONS)
1236
1237         elif n.type == syms.decorator:
1238             if len(n.children) > 1 and not is_simple_decorator_expression(
1239                 n.children[1]
1240             ):
1241                 features.add(Feature.RELAXED_DECORATORS)
1242
1243         elif (
1244             n.type in {syms.typedargslist, syms.arglist}
1245             and n.children
1246             and n.children[-1].type == token.COMMA
1247         ):
1248             if n.type == syms.typedargslist:
1249                 feature = Feature.TRAILING_COMMA_IN_DEF
1250             else:
1251                 feature = Feature.TRAILING_COMMA_IN_CALL
1252
1253             for ch in n.children:
1254                 if ch.type in STARS:
1255                     features.add(feature)
1256
1257                 if ch.type == syms.argument:
1258                     for argch in ch.children:
1259                         if argch.type in STARS:
1260                             features.add(feature)
1261
1262         elif (
1263             n.type in {syms.return_stmt, syms.yield_expr}
1264             and len(n.children) >= 2
1265             and n.children[1].type == syms.testlist_star_expr
1266             and any(child.type == syms.star_expr for child in n.children[1].children)
1267         ):
1268             features.add(Feature.UNPACKING_ON_FLOW)
1269
1270         elif (
1271             n.type == syms.annassign
1272             and len(n.children) >= 4
1273             and n.children[3].type == syms.testlist_star_expr
1274         ):
1275             features.add(Feature.ANN_ASSIGN_EXTENDED_RHS)
1276
1277     return features
1278
1279
1280 def detect_target_versions(
1281     node: Node, *, future_imports: Optional[Set[str]] = None
1282 ) -> Set[TargetVersion]:
1283     """Detect the version to target based on the nodes used."""
1284     features = get_features_used(node, future_imports=future_imports)
1285     return {
1286         version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
1287     }
1288
1289
1290 def get_future_imports(node: Node) -> Set[str]:
1291     """Return a set of __future__ imports in the file."""
1292     imports: Set[str] = set()
1293
1294     def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
1295         for child in children:
1296             if isinstance(child, Leaf):
1297                 if child.type == token.NAME:
1298                     yield child.value
1299
1300             elif child.type == syms.import_as_name:
1301                 orig_name = child.children[0]
1302                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
1303                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
1304                 yield orig_name.value
1305
1306             elif child.type == syms.import_as_names:
1307                 yield from get_imports_from_children(child.children)
1308
1309             else:
1310                 raise AssertionError("Invalid syntax parsing imports")
1311
1312     for child in node.children:
1313         if child.type != syms.simple_stmt:
1314             break
1315
1316         first_child = child.children[0]
1317         if isinstance(first_child, Leaf):
1318             # Continue looking if we see a docstring; otherwise stop.
1319             if (
1320                 len(child.children) == 2
1321                 and first_child.type == token.STRING
1322                 and child.children[1].type == token.NEWLINE
1323             ):
1324                 continue
1325
1326             break
1327
1328         elif first_child.type == syms.import_from:
1329             module_name = first_child.children[1]
1330             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
1331                 break
1332
1333             imports |= set(get_imports_from_children(first_child.children[3:]))
1334         else:
1335             break
1336
1337     return imports
1338
1339
1340 def assert_equivalent(src: str, dst: str) -> None:
1341     """Raise AssertionError if `src` and `dst` aren't equivalent."""
1342     try:
1343         src_ast = parse_ast(src)
1344     except Exception as exc:
1345         raise AssertionError(
1346             f"cannot use --safe with this file; failed to parse source file AST: "
1347             f"{exc}\n"
1348             f"This could be caused by running Black with an older Python version "
1349             f"that does not support new syntax used in your source file."
1350         ) from exc
1351
1352     try:
1353         dst_ast = parse_ast(dst)
1354     except Exception as exc:
1355         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
1356         raise AssertionError(
1357             f"INTERNAL ERROR: Black produced invalid code: {exc}. "
1358             "Please report a bug on https://github.com/psf/black/issues.  "
1359             f"This invalid output might be helpful: {log}"
1360         ) from None
1361
1362     src_ast_str = "\n".join(stringify_ast(src_ast))
1363     dst_ast_str = "\n".join(stringify_ast(dst_ast))
1364     if src_ast_str != dst_ast_str:
1365         log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1366         raise AssertionError(
1367             "INTERNAL ERROR: Black produced code that is not equivalent to the"
1368             f" source.  Please report a bug on "
1369             f"https://github.com/psf/black/issues.  This diff might be helpful: {log}"
1370         ) from None
1371
1372
1373 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1374     """Raise AssertionError if `dst` reformats differently the second time."""
1375     # We shouldn't call format_str() here, because that formats the string
1376     # twice and may hide a bug where we bounce back and forth between two
1377     # versions.
1378     newdst = _format_str_once(dst, mode=mode)
1379     if dst != newdst:
1380         log = dump_to_file(
1381             str(mode),
1382             diff(src, dst, "source", "first pass"),
1383             diff(dst, newdst, "first pass", "second pass"),
1384         )
1385         raise AssertionError(
1386             "INTERNAL ERROR: Black produced different code on the second pass of the"
1387             " formatter.  Please report a bug on https://github.com/psf/black/issues."
1388             f"  This diff might be helpful: {log}"
1389         ) from None
1390
1391
1392 @contextmanager
1393 def nullcontext() -> Iterator[None]:
1394     """Return an empty context manager.
1395
1396     To be used like `nullcontext` in Python 3.7.
1397     """
1398     yield
1399
1400
1401 def patch_click() -> None:
1402     """Make Click not crash on Python 3.6 with LANG=C.
1403
1404     On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1405     default which restricts paths that it can access during the lifetime of the
1406     application.  Click refuses to work in this scenario by raising a RuntimeError.
1407
1408     In case of Black the likelihood that non-ASCII characters are going to be used in
1409     file paths is minimal since it's Python source code.  Moreover, this crash was
1410     spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1411     """
1412     try:
1413         from click import core
1414         from click import _unicodefun
1415     except ModuleNotFoundError:
1416         return
1417
1418     for module in (core, _unicodefun):
1419         if hasattr(module, "_verify_python3_env"):
1420             module._verify_python3_env = lambda: None  # type: ignore
1421         if hasattr(module, "_verify_python_env"):
1422             module._verify_python_env = lambda: None  # type: ignore
1423
1424
1425 def patched_main() -> None:
1426     maybe_install_uvloop()
1427     freeze_support()
1428     patch_click()
1429     main()
1430
1431
1432 if __name__ == "__main__":
1433     patched_main()