]> git.madduck.net Git - etc/vim.git/blob - src/black/__init__.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

d95e9b13bb92b80cdab097c2ec840d4fbf13d52a
[etc/vim.git] / src / black / __init__.py
1 import asyncio
2 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
3 from contextlib import contextmanager
4 from datetime import datetime
5 from enum import Enum
6 import io
7 from multiprocessing import Manager, freeze_support
8 import os
9 from pathlib import Path
10 import regex as re
11 import signal
12 import sys
13 import tokenize
14 import traceback
15 from typing import (
16     Any,
17     Dict,
18     Generator,
19     Iterator,
20     List,
21     Optional,
22     Pattern,
23     Set,
24     Sized,
25     Tuple,
26     Union,
27 )
28
29 from dataclasses import replace
30 import click
31
32 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
33 from black.const import STDIN_PLACEHOLDER
34 from black.nodes import STARS, syms, is_simple_decorator_expression
35 from black.lines import Line, EmptyLineTracker
36 from black.linegen import transform_line, LineGenerator, LN
37 from black.comments import normalize_fmt_off
38 from black.mode import Mode, TargetVersion
39 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
40 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
41 from black.concurrency import cancel, shutdown, maybe_install_uvloop
42 from black.output import dump_to_file, diff, color_diff, out, err
43 from black.report import Report, Changed
44 from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml
45 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
46 from black.files import wrap_stream_for_windows
47 from black.parsing import InvalidInput  # noqa F401
48 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
49
50
51 # lib2to3 fork
52 from blib2to3.pytree import Node, Leaf
53 from blib2to3.pgen2 import token
54
55 from _black_version import version as __version__
56
57 # types
58 FileContent = str
59 Encoding = str
60 NewLine = str
61
62
63 class NothingChanged(UserWarning):
64     """Raised when reformatted code is the same as source."""
65
66
67 class WriteBack(Enum):
68     NO = 0
69     YES = 1
70     DIFF = 2
71     CHECK = 3
72     COLOR_DIFF = 4
73
74     @classmethod
75     def from_configuration(
76         cls, *, check: bool, diff: bool, color: bool = False
77     ) -> "WriteBack":
78         if check and not diff:
79             return cls.CHECK
80
81         if diff and color:
82             return cls.COLOR_DIFF
83
84         return cls.DIFF if diff else cls.YES
85
86
87 # Legacy name, left for integrations.
88 FileMode = Mode
89
90
91 def read_pyproject_toml(
92     ctx: click.Context, param: click.Parameter, value: Optional[str]
93 ) -> Optional[str]:
94     """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
95
96     Returns the path to a successfully found and read configuration file, None
97     otherwise.
98     """
99     if not value:
100         value = find_pyproject_toml(ctx.params.get("src", ()))
101         if value is None:
102             return None
103
104     try:
105         config = parse_pyproject_toml(value)
106     except (OSError, ValueError) as e:
107         raise click.FileError(
108             filename=value, hint=f"Error reading configuration file: {e}"
109         )
110
111     if not config:
112         return None
113     else:
114         # Sanitize the values to be Click friendly. For more information please see:
115         # https://github.com/psf/black/issues/1458
116         # https://github.com/pallets/click/issues/1567
117         config = {
118             k: str(v) if not isinstance(v, (list, dict)) else v
119             for k, v in config.items()
120         }
121
122     target_version = config.get("target_version")
123     if target_version is not None and not isinstance(target_version, list):
124         raise click.BadOptionUsage(
125             "target-version", "Config key target-version must be a list"
126         )
127
128     default_map: Dict[str, Any] = {}
129     if ctx.default_map:
130         default_map.update(ctx.default_map)
131     default_map.update(config)
132
133     ctx.default_map = default_map
134     return value
135
136
137 def target_version_option_callback(
138     c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
139 ) -> List[TargetVersion]:
140     """Compute the target versions from a --target-version flag.
141
142     This is its own function because mypy couldn't infer the type correctly
143     when it was a lambda, causing mypyc trouble.
144     """
145     return [TargetVersion[val.upper()] for val in v]
146
147
148 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
149     """Compile a regular expression string in `regex`.
150
151     If it contains newlines, use verbose mode.
152     """
153     if "\n" in regex:
154         regex = "(?x)" + regex
155     compiled: Pattern[str] = re.compile(regex)
156     return compiled
157
158
159 def validate_regex(
160     ctx: click.Context,
161     param: click.Parameter,
162     value: Optional[str],
163 ) -> Optional[Pattern]:
164     try:
165         return re_compile_maybe_verbose(value) if value is not None else None
166     except re.error:
167         raise click.BadParameter("Not a valid regular expression")
168
169
170 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
171 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
172 @click.option(
173     "-l",
174     "--line-length",
175     type=int,
176     default=DEFAULT_LINE_LENGTH,
177     help="How many characters per line to allow.",
178     show_default=True,
179 )
180 @click.option(
181     "-t",
182     "--target-version",
183     type=click.Choice([v.name.lower() for v in TargetVersion]),
184     callback=target_version_option_callback,
185     multiple=True,
186     help=(
187         "Python versions that should be supported by Black's output. [default: per-file"
188         " auto-detection]"
189     ),
190 )
191 @click.option(
192     "--pyi",
193     is_flag=True,
194     help=(
195         "Format all input files like typing stubs regardless of file extension (useful"
196         " when piping source on standard input)."
197     ),
198 )
199 @click.option(
200     "-S",
201     "--skip-string-normalization",
202     is_flag=True,
203     help="Don't normalize string quotes or prefixes.",
204 )
205 @click.option(
206     "-C",
207     "--skip-magic-trailing-comma",
208     is_flag=True,
209     help="Don't use trailing commas as a reason to split lines.",
210 )
211 @click.option(
212     "--experimental-string-processing",
213     is_flag=True,
214     hidden=True,
215     help=(
216         "Experimental option that performs more normalization on string literals."
217         " Currently disabled because it leads to some crashes."
218     ),
219 )
220 @click.option(
221     "--check",
222     is_flag=True,
223     help=(
224         "Don't write the files back, just return the status. Return code 0 means"
225         " nothing would change. Return code 1 means some files would be reformatted."
226         " Return code 123 means there was an internal error."
227     ),
228 )
229 @click.option(
230     "--diff",
231     is_flag=True,
232     help="Don't write the files back, just output a diff for each file on stdout.",
233 )
234 @click.option(
235     "--color/--no-color",
236     is_flag=True,
237     help="Show colored diff. Only applies when `--diff` is given.",
238 )
239 @click.option(
240     "--fast/--safe",
241     is_flag=True,
242     help="If --fast given, skip temporary sanity checks. [default: --safe]",
243 )
244 @click.option(
245     "--include",
246     type=str,
247     default=DEFAULT_INCLUDES,
248     callback=validate_regex,
249     help=(
250         "A regular expression that matches files and directories that should be"
251         " included on recursive searches. An empty value means all files are included"
252         " regardless of the name. Use forward slashes for directories on all platforms"
253         " (Windows, too). Exclusions are calculated first, inclusions later."
254     ),
255     show_default=True,
256 )
257 @click.option(
258     "--exclude",
259     type=str,
260     callback=validate_regex,
261     help=(
262         "A regular expression that matches files and directories that should be"
263         " excluded on recursive searches. An empty value means no paths are excluded."
264         " Use forward slashes for directories on all platforms (Windows, too)."
265         " Exclusions are calculated first, inclusions later. [default:"
266         f" {DEFAULT_EXCLUDES}]"
267     ),
268     show_default=False,
269 )
270 @click.option(
271     "--extend-exclude",
272     type=str,
273     callback=validate_regex,
274     help=(
275         "Like --exclude, but adds additional files and directories on top of the"
276         " excluded ones. (Useful if you simply want to add to the default)"
277     ),
278 )
279 @click.option(
280     "--force-exclude",
281     type=str,
282     callback=validate_regex,
283     help=(
284         "Like --exclude, but files and directories matching this regex will be "
285         "excluded even when they are passed explicitly as arguments."
286     ),
287 )
288 @click.option(
289     "--stdin-filename",
290     type=str,
291     help=(
292         "The name of the file when passing it through stdin. Useful to make "
293         "sure Black will respect --force-exclude option on some "
294         "editors that rely on using stdin."
295     ),
296 )
297 @click.option(
298     "-q",
299     "--quiet",
300     is_flag=True,
301     help=(
302         "Don't emit non-error messages to stderr. Errors are still emitted; silence"
303         " those with 2>/dev/null."
304     ),
305 )
306 @click.option(
307     "-v",
308     "--verbose",
309     is_flag=True,
310     help=(
311         "Also emit messages to stderr about files that were not changed or were ignored"
312         " due to exclusion patterns."
313     ),
314 )
315 @click.version_option(version=__version__)
316 @click.argument(
317     "src",
318     nargs=-1,
319     type=click.Path(
320         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
321     ),
322     is_eager=True,
323 )
324 @click.option(
325     "--config",
326     type=click.Path(
327         exists=True,
328         file_okay=True,
329         dir_okay=False,
330         readable=True,
331         allow_dash=False,
332         path_type=str,
333     ),
334     is_eager=True,
335     callback=read_pyproject_toml,
336     help="Read configuration from FILE path.",
337 )
338 @click.pass_context
339 def main(
340     ctx: click.Context,
341     code: Optional[str],
342     line_length: int,
343     target_version: List[TargetVersion],
344     check: bool,
345     diff: bool,
346     color: bool,
347     fast: bool,
348     pyi: bool,
349     skip_string_normalization: bool,
350     skip_magic_trailing_comma: bool,
351     experimental_string_processing: bool,
352     quiet: bool,
353     verbose: bool,
354     include: Pattern,
355     exclude: Optional[Pattern],
356     extend_exclude: Optional[Pattern],
357     force_exclude: Optional[Pattern],
358     stdin_filename: Optional[str],
359     src: Tuple[str, ...],
360     config: Optional[str],
361 ) -> None:
362     """The uncompromising code formatter."""
363     write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
364     if target_version:
365         versions = set(target_version)
366     else:
367         # We'll autodetect later.
368         versions = set()
369     mode = Mode(
370         target_versions=versions,
371         line_length=line_length,
372         is_pyi=pyi,
373         string_normalization=not skip_string_normalization,
374         magic_trailing_comma=not skip_magic_trailing_comma,
375         experimental_string_processing=experimental_string_processing,
376     )
377     if config and verbose:
378         out(f"Using configuration from {config}.", bold=False, fg="blue")
379
380     if code is not None:
381         # Run in quiet mode by default with -c; the extra output isn't useful.
382         # You can still pass -v to get verbose output.
383         quiet = True
384
385     report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
386
387     if code is not None:
388         reformat_code(
389             content=code, fast=fast, write_back=write_back, mode=mode, report=report
390         )
391     else:
392         sources = get_sources(
393             ctx=ctx,
394             src=src,
395             quiet=quiet,
396             verbose=verbose,
397             include=include,
398             exclude=exclude,
399             extend_exclude=extend_exclude,
400             force_exclude=force_exclude,
401             report=report,
402             stdin_filename=stdin_filename,
403         )
404
405         path_empty(
406             sources,
407             "No Python files are present to be formatted. Nothing to do 😴",
408             quiet,
409             verbose,
410             ctx,
411         )
412
413         if len(sources) == 1:
414             reformat_one(
415                 src=sources.pop(),
416                 fast=fast,
417                 write_back=write_back,
418                 mode=mode,
419                 report=report,
420             )
421         else:
422             reformat_many(
423                 sources=sources,
424                 fast=fast,
425                 write_back=write_back,
426                 mode=mode,
427                 report=report,
428             )
429
430     if verbose or not quiet:
431         out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
432         if code is None:
433             click.secho(str(report), err=True)
434     ctx.exit(report.return_code)
435
436
437 def get_sources(
438     *,
439     ctx: click.Context,
440     src: Tuple[str, ...],
441     quiet: bool,
442     verbose: bool,
443     include: Pattern[str],
444     exclude: Optional[Pattern[str]],
445     extend_exclude: Optional[Pattern[str]],
446     force_exclude: Optional[Pattern[str]],
447     report: "Report",
448     stdin_filename: Optional[str],
449 ) -> Set[Path]:
450     """Compute the set of files to be formatted."""
451
452     root = find_project_root(src)
453     sources: Set[Path] = set()
454     path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx)
455
456     if exclude is None:
457         exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
458         gitignore = get_gitignore(root)
459     else:
460         gitignore = None
461
462     for s in src:
463         if s == "-" and stdin_filename:
464             p = Path(stdin_filename)
465             is_stdin = True
466         else:
467             p = Path(s)
468             is_stdin = False
469
470         if is_stdin or p.is_file():
471             normalized_path = normalize_path_maybe_ignore(p, root, report)
472             if normalized_path is None:
473                 continue
474
475             normalized_path = "/" + normalized_path
476             # Hard-exclude any files that matches the `--force-exclude` regex.
477             if force_exclude:
478                 force_exclude_match = force_exclude.search(normalized_path)
479             else:
480                 force_exclude_match = None
481             if force_exclude_match and force_exclude_match.group(0):
482                 report.path_ignored(p, "matches the --force-exclude regular expression")
483                 continue
484
485             if is_stdin:
486                 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
487
488             sources.add(p)
489         elif p.is_dir():
490             sources.update(
491                 gen_python_files(
492                     p.iterdir(),
493                     root,
494                     include,
495                     exclude,
496                     extend_exclude,
497                     force_exclude,
498                     report,
499                     gitignore,
500                 )
501             )
502         elif s == "-":
503             sources.add(p)
504         else:
505             err(f"invalid path: {s}")
506     return sources
507
508
509 def path_empty(
510     src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
511 ) -> None:
512     """
513     Exit if there is no `src` provided for formatting
514     """
515     if not src:
516         if verbose or not quiet:
517             out(msg)
518         ctx.exit(0)
519
520
521 def reformat_code(
522     content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report
523 ) -> None:
524     """
525     Reformat and print out `content` without spawning child processes.
526     Similar to `reformat_one`, but for string content.
527
528     `fast`, `write_back`, and `mode` options are passed to
529     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
530     """
531     path = Path("<string>")
532     try:
533         changed = Changed.NO
534         if format_stdin_to_stdout(
535             content=content, fast=fast, write_back=write_back, mode=mode
536         ):
537             changed = Changed.YES
538         report.done(path, changed)
539     except Exception as exc:
540         if report.verbose:
541             traceback.print_exc()
542         report.failed(path, str(exc))
543
544
545 def reformat_one(
546     src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
547 ) -> None:
548     """Reformat a single file under `src` without spawning child processes.
549
550     `fast`, `write_back`, and `mode` options are passed to
551     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
552     """
553     try:
554         changed = Changed.NO
555
556         if str(src) == "-":
557             is_stdin = True
558         elif str(src).startswith(STDIN_PLACEHOLDER):
559             is_stdin = True
560             # Use the original name again in case we want to print something
561             # to the user
562             src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
563         else:
564             is_stdin = False
565
566         if is_stdin:
567             if src.suffix == ".pyi":
568                 mode = replace(mode, is_pyi=True)
569             if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
570                 changed = Changed.YES
571         else:
572             cache: Cache = {}
573             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
574                 cache = read_cache(mode)
575                 res_src = src.resolve()
576                 res_src_s = str(res_src)
577                 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
578                     changed = Changed.CACHED
579             if changed is not Changed.CACHED and format_file_in_place(
580                 src, fast=fast, write_back=write_back, mode=mode
581             ):
582                 changed = Changed.YES
583             if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
584                 write_back is WriteBack.CHECK and changed is Changed.NO
585             ):
586                 write_cache(cache, [src], mode)
587         report.done(src, changed)
588     except Exception as exc:
589         if report.verbose:
590             traceback.print_exc()
591         report.failed(src, str(exc))
592
593
594 def reformat_many(
595     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
596 ) -> None:
597     """Reformat multiple files using a ProcessPoolExecutor."""
598     executor: Executor
599     loop = asyncio.get_event_loop()
600     worker_count = os.cpu_count()
601     if sys.platform == "win32":
602         # Work around https://bugs.python.org/issue26903
603         worker_count = min(worker_count, 60)
604     try:
605         executor = ProcessPoolExecutor(max_workers=worker_count)
606     except (ImportError, OSError):
607         # we arrive here if the underlying system does not support multi-processing
608         # like in AWS Lambda or Termux, in which case we gracefully fallback to
609         # a ThreadPoolExecutor with just a single worker (more workers would not do us
610         # any good due to the Global Interpreter Lock)
611         executor = ThreadPoolExecutor(max_workers=1)
612
613     try:
614         loop.run_until_complete(
615             schedule_formatting(
616                 sources=sources,
617                 fast=fast,
618                 write_back=write_back,
619                 mode=mode,
620                 report=report,
621                 loop=loop,
622                 executor=executor,
623             )
624         )
625     finally:
626         shutdown(loop)
627         if executor is not None:
628             executor.shutdown()
629
630
631 async def schedule_formatting(
632     sources: Set[Path],
633     fast: bool,
634     write_back: WriteBack,
635     mode: Mode,
636     report: "Report",
637     loop: asyncio.AbstractEventLoop,
638     executor: Executor,
639 ) -> None:
640     """Run formatting of `sources` in parallel using the provided `executor`.
641
642     (Use ProcessPoolExecutors for actual parallelism.)
643
644     `write_back`, `fast`, and `mode` options are passed to
645     :func:`format_file_in_place`.
646     """
647     cache: Cache = {}
648     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
649         cache = read_cache(mode)
650         sources, cached = filter_cached(cache, sources)
651         for src in sorted(cached):
652             report.done(src, Changed.CACHED)
653     if not sources:
654         return
655
656     cancelled = []
657     sources_to_cache = []
658     lock = None
659     if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
660         # For diff output, we need locks to ensure we don't interleave output
661         # from different processes.
662         manager = Manager()
663         lock = manager.Lock()
664     tasks = {
665         asyncio.ensure_future(
666             loop.run_in_executor(
667                 executor, format_file_in_place, src, fast, mode, write_back, lock
668             )
669         ): src
670         for src in sorted(sources)
671     }
672     pending = tasks.keys()
673     try:
674         loop.add_signal_handler(signal.SIGINT, cancel, pending)
675         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
676     except NotImplementedError:
677         # There are no good alternatives for these on Windows.
678         pass
679     while pending:
680         done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
681         for task in done:
682             src = tasks.pop(task)
683             if task.cancelled():
684                 cancelled.append(task)
685             elif task.exception():
686                 report.failed(src, str(task.exception()))
687             else:
688                 changed = Changed.YES if task.result() else Changed.NO
689                 # If the file was written back or was successfully checked as
690                 # well-formatted, store this information in the cache.
691                 if write_back is WriteBack.YES or (
692                     write_back is WriteBack.CHECK and changed is Changed.NO
693                 ):
694                     sources_to_cache.append(src)
695                 report.done(src, changed)
696     if cancelled:
697         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
698     if sources_to_cache:
699         write_cache(cache, sources_to_cache, mode)
700
701
702 def format_file_in_place(
703     src: Path,
704     fast: bool,
705     mode: Mode,
706     write_back: WriteBack = WriteBack.NO,
707     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
708 ) -> bool:
709     """Format file under `src` path. Return True if changed.
710
711     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
712     code to the file.
713     `mode` and `fast` options are passed to :func:`format_file_contents`.
714     """
715     if src.suffix == ".pyi":
716         mode = replace(mode, is_pyi=True)
717
718     then = datetime.utcfromtimestamp(src.stat().st_mtime)
719     with open(src, "rb") as buf:
720         src_contents, encoding, newline = decode_bytes(buf.read())
721     try:
722         dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
723     except NothingChanged:
724         return False
725
726     if write_back == WriteBack.YES:
727         with open(src, "w", encoding=encoding, newline=newline) as f:
728             f.write(dst_contents)
729     elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
730         now = datetime.utcnow()
731         src_name = f"{src}\t{then} +0000"
732         dst_name = f"{src}\t{now} +0000"
733         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
734
735         if write_back == WriteBack.COLOR_DIFF:
736             diff_contents = color_diff(diff_contents)
737
738         with lock or nullcontext():
739             f = io.TextIOWrapper(
740                 sys.stdout.buffer,
741                 encoding=encoding,
742                 newline=newline,
743                 write_through=True,
744             )
745             f = wrap_stream_for_windows(f)
746             f.write(diff_contents)
747             f.detach()
748
749     return True
750
751
752 def format_stdin_to_stdout(
753     fast: bool,
754     *,
755     content: Optional[str] = None,
756     write_back: WriteBack = WriteBack.NO,
757     mode: Mode,
758 ) -> bool:
759     """Format file on stdin. Return True if changed.
760
761     If content is None, it's read from sys.stdin.
762
763     If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
764     write a diff to stdout. The `mode` argument is passed to
765     :func:`format_file_contents`.
766     """
767     then = datetime.utcnow()
768
769     if content is None:
770         src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
771     else:
772         src, encoding, newline = content, "utf-8", ""
773
774     dst = src
775     try:
776         dst = format_file_contents(src, fast=fast, mode=mode)
777         return True
778
779     except NothingChanged:
780         return False
781
782     finally:
783         f = io.TextIOWrapper(
784             sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
785         )
786         if write_back == WriteBack.YES:
787             # Make sure there's a newline after the content
788             dst += "" if dst[-1] == "\n" else "\n"
789             f.write(dst)
790         elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
791             now = datetime.utcnow()
792             src_name = f"STDIN\t{then} +0000"
793             dst_name = f"STDOUT\t{now} +0000"
794             d = diff(src, dst, src_name, dst_name)
795             if write_back == WriteBack.COLOR_DIFF:
796                 d = color_diff(d)
797                 f = wrap_stream_for_windows(f)
798             f.write(d)
799         f.detach()
800
801
802 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
803     """Reformat contents of a file and return new contents.
804
805     If `fast` is False, additionally confirm that the reformatted code is
806     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
807     `mode` is passed to :func:`format_str`.
808     """
809     if not src_contents.strip():
810         raise NothingChanged
811
812     dst_contents = format_str(src_contents, mode=mode)
813     if src_contents == dst_contents:
814         raise NothingChanged
815
816     if not fast:
817         assert_equivalent(src_contents, dst_contents)
818
819         # Forced second pass to work around optional trailing commas (becoming
820         # forced trailing commas on pass 2) interacting differently with optional
821         # parentheses.  Admittedly ugly.
822         dst_contents_pass2 = format_str(dst_contents, mode=mode)
823         if dst_contents != dst_contents_pass2:
824             dst_contents = dst_contents_pass2
825             assert_equivalent(src_contents, dst_contents, pass_num=2)
826             assert_stable(src_contents, dst_contents, mode=mode)
827         # Note: no need to explicitly call `assert_stable` if `dst_contents` was
828         # the same as `dst_contents_pass2`.
829     return dst_contents
830
831
832 def format_str(src_contents: str, *, mode: Mode) -> FileContent:
833     """Reformat a string and return new contents.
834
835     `mode` determines formatting options, such as how many characters per line are
836     allowed.  Example:
837
838     >>> import black
839     >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
840     def f(arg: str = "") -> None:
841         ...
842
843     A more complex example:
844
845     >>> print(
846     ...   black.format_str(
847     ...     "def f(arg:str='')->None: hey",
848     ...     mode=black.Mode(
849     ...       target_versions={black.TargetVersion.PY36},
850     ...       line_length=10,
851     ...       string_normalization=False,
852     ...       is_pyi=False,
853     ...     ),
854     ...   ),
855     ... )
856     def f(
857         arg: str = '',
858     ) -> None:
859         hey
860
861     """
862     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
863     dst_contents = []
864     future_imports = get_future_imports(src_node)
865     if mode.target_versions:
866         versions = mode.target_versions
867     else:
868         versions = detect_target_versions(src_node)
869     normalize_fmt_off(src_node)
870     lines = LineGenerator(
871         mode=mode,
872         remove_u_prefix="unicode_literals" in future_imports
873         or supports_feature(versions, Feature.UNICODE_LITERALS),
874     )
875     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
876     empty_line = Line(mode=mode)
877     after = 0
878     split_line_features = {
879         feature
880         for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
881         if supports_feature(versions, feature)
882     }
883     for current_line in lines.visit(src_node):
884         dst_contents.append(str(empty_line) * after)
885         before, after = elt.maybe_empty_lines(current_line)
886         dst_contents.append(str(empty_line) * before)
887         for line in transform_line(
888             current_line, mode=mode, features=split_line_features
889         ):
890             dst_contents.append(str(line))
891     return "".join(dst_contents)
892
893
894 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
895     """Return a tuple of (decoded_contents, encoding, newline).
896
897     `newline` is either CRLF or LF but `decoded_contents` is decoded with
898     universal newlines (i.e. only contains LF).
899     """
900     srcbuf = io.BytesIO(src)
901     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
902     if not lines:
903         return "", encoding, "\n"
904
905     newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
906     srcbuf.seek(0)
907     with io.TextIOWrapper(srcbuf, encoding) as tiow:
908         return tiow.read(), encoding, newline
909
910
911 def get_features_used(node: Node) -> Set[Feature]:
912     """Return a set of (relatively) new Python features used in this file.
913
914     Currently looking for:
915     - f-strings;
916     - underscores in numeric literals;
917     - trailing commas after * or ** in function signatures and calls;
918     - positional only arguments in function signatures and lambdas;
919     - assignment expression;
920     - relaxed decorator syntax;
921     """
922     features: Set[Feature] = set()
923     for n in node.pre_order():
924         if n.type == token.STRING:
925             value_head = n.value[:2]  # type: ignore
926             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
927                 features.add(Feature.F_STRINGS)
928
929         elif n.type == token.NUMBER:
930             if "_" in n.value:  # type: ignore
931                 features.add(Feature.NUMERIC_UNDERSCORES)
932
933         elif n.type == token.SLASH:
934             if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
935                 features.add(Feature.POS_ONLY_ARGUMENTS)
936
937         elif n.type == token.COLONEQUAL:
938             features.add(Feature.ASSIGNMENT_EXPRESSIONS)
939
940         elif n.type == syms.decorator:
941             if len(n.children) > 1 and not is_simple_decorator_expression(
942                 n.children[1]
943             ):
944                 features.add(Feature.RELAXED_DECORATORS)
945
946         elif (
947             n.type in {syms.typedargslist, syms.arglist}
948             and n.children
949             and n.children[-1].type == token.COMMA
950         ):
951             if n.type == syms.typedargslist:
952                 feature = Feature.TRAILING_COMMA_IN_DEF
953             else:
954                 feature = Feature.TRAILING_COMMA_IN_CALL
955
956             for ch in n.children:
957                 if ch.type in STARS:
958                     features.add(feature)
959
960                 if ch.type == syms.argument:
961                     for argch in ch.children:
962                         if argch.type in STARS:
963                             features.add(feature)
964
965     return features
966
967
968 def detect_target_versions(node: Node) -> Set[TargetVersion]:
969     """Detect the version to target based on the nodes used."""
970     features = get_features_used(node)
971     return {
972         version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
973     }
974
975
976 def get_future_imports(node: Node) -> Set[str]:
977     """Return a set of __future__ imports in the file."""
978     imports: Set[str] = set()
979
980     def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
981         for child in children:
982             if isinstance(child, Leaf):
983                 if child.type == token.NAME:
984                     yield child.value
985
986             elif child.type == syms.import_as_name:
987                 orig_name = child.children[0]
988                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
989                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
990                 yield orig_name.value
991
992             elif child.type == syms.import_as_names:
993                 yield from get_imports_from_children(child.children)
994
995             else:
996                 raise AssertionError("Invalid syntax parsing imports")
997
998     for child in node.children:
999         if child.type != syms.simple_stmt:
1000             break
1001
1002         first_child = child.children[0]
1003         if isinstance(first_child, Leaf):
1004             # Continue looking if we see a docstring; otherwise stop.
1005             if (
1006                 len(child.children) == 2
1007                 and first_child.type == token.STRING
1008                 and child.children[1].type == token.NEWLINE
1009             ):
1010                 continue
1011
1012             break
1013
1014         elif first_child.type == syms.import_from:
1015             module_name = first_child.children[1]
1016             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
1017                 break
1018
1019             imports |= set(get_imports_from_children(first_child.children[3:]))
1020         else:
1021             break
1022
1023     return imports
1024
1025
1026 def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None:
1027     """Raise AssertionError if `src` and `dst` aren't equivalent."""
1028     try:
1029         src_ast = parse_ast(src)
1030     except Exception as exc:
1031         raise AssertionError(
1032             "cannot use --safe with this file; failed to parse source file.  AST"
1033             f" error message: {exc}"
1034         )
1035
1036     try:
1037         dst_ast = parse_ast(dst)
1038     except Exception as exc:
1039         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
1040         raise AssertionError(
1041             f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. "
1042             "Please report a bug on https://github.com/psf/black/issues.  "
1043             f"This invalid output might be helpful: {log}"
1044         ) from None
1045
1046     src_ast_str = "\n".join(stringify_ast(src_ast))
1047     dst_ast_str = "\n".join(stringify_ast(dst_ast))
1048     if src_ast_str != dst_ast_str:
1049         log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1050         raise AssertionError(
1051             "INTERNAL ERROR: Black produced code that is not equivalent to the"
1052             f" source on pass {pass_num}.  Please report a bug on "
1053             f"https://github.com/psf/black/issues.  This diff might be helpful: {log}"
1054         ) from None
1055
1056
1057 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1058     """Raise AssertionError if `dst` reformats differently the second time."""
1059     newdst = format_str(dst, mode=mode)
1060     if dst != newdst:
1061         log = dump_to_file(
1062             str(mode),
1063             diff(src, dst, "source", "first pass"),
1064             diff(dst, newdst, "first pass", "second pass"),
1065         )
1066         raise AssertionError(
1067             "INTERNAL ERROR: Black produced different code on the second pass of the"
1068             " formatter.  Please report a bug on https://github.com/psf/black/issues."
1069             f"  This diff might be helpful: {log}"
1070         ) from None
1071
1072
1073 @contextmanager
1074 def nullcontext() -> Iterator[None]:
1075     """Return an empty context manager.
1076
1077     To be used like `nullcontext` in Python 3.7.
1078     """
1079     yield
1080
1081
1082 def patch_click() -> None:
1083     """Make Click not crash on Python 3.6 with LANG=C.
1084
1085     On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1086     default which restricts paths that it can access during the lifetime of the
1087     application.  Click refuses to work in this scenario by raising a RuntimeError.
1088
1089     In case of Black the likelihood that non-ASCII characters are going to be used in
1090     file paths is minimal since it's Python source code.  Moreover, this crash was
1091     spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1092     """
1093     try:
1094         from click import core
1095         from click import _unicodefun  # type: ignore
1096     except ModuleNotFoundError:
1097         return
1098
1099     for module in (core, _unicodefun):
1100         if hasattr(module, "_verify_python3_env"):
1101             module._verify_python3_env = lambda: None  # type: ignore
1102         if hasattr(module, "_verify_python_env"):
1103             module._verify_python_env = lambda: None  # type: ignore
1104
1105
1106 def patched_main() -> None:
1107     maybe_install_uvloop()
1108     freeze_support()
1109     patch_click()
1110     main()
1111
1112
1113 if __name__ == "__main__":
1114     patched_main()