src/black/__init__.py

   1 import asyncio
   2 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
   3 from contextlib import contextmanager
   4 from datetime import datetime
   5 from enum import Enum
   6 import io
   7 from multiprocessing import Manager, freeze_support
   8 import os
   9 from pathlib import Path
  10 import regex as re
  11 import signal
  12 import sys
  13 import tokenize
  14 import traceback
  15 from typing import (
  16     Any,
  17     Dict,
  18     Generator,
  19     Iterator,
  20     List,
  21     Optional,
  22     Pattern,
  23     Set,
  24     Sized,
  25     Tuple,
  26     Union,
  27 )
  28
  29 from dataclasses import replace
  30 import click
  31
  32 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
  33 from black.const import STDIN_PLACEHOLDER
  34 from black.nodes import STARS, syms, is_simple_decorator_expression
  35 from black.lines import Line, EmptyLineTracker
  36 from black.linegen import transform_line, LineGenerator, LN
  37 from black.comments import normalize_fmt_off
  38 from black.mode import Mode, TargetVersion
  39 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
  40 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
  41 from black.concurrency import cancel, shutdown
  42 from black.output import dump_to_file, diff, color_diff, out, err
  43 from black.report import Report, Changed
  44 from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml
  45 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
  46 from black.files import wrap_stream_for_windows
  47 from black.parsing import InvalidInput  # noqa F401
  48 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
  49
  50
  51 # lib2to3 fork
  52 from blib2to3.pytree import Node, Leaf
  53 from blib2to3.pgen2 import token
  54
  55 from _black_version import version as __version__
  56
  57 # If our environment has uvloop installed lets use it
  58 try:
  59     import uvloop
  60
  61     uvloop.install()
  62 except ImportError:
  63     pass
  64
  65 # types
  66 FileContent = str
  67 Encoding = str
  68 NewLine = str
  69
  70
  71 class NothingChanged(UserWarning):
  72     """Raised when reformatted code is the same as source."""
  73
  74
  75 class WriteBack(Enum):
  76     NO = 0
  77     YES = 1
  78     DIFF = 2
  79     CHECK = 3
  80     COLOR_DIFF = 4
  81
  82     @classmethod
  83     def from_configuration(
  84         cls, *, check: bool, diff: bool, color: bool = False
  85     ) -> "WriteBack":
  86         if check and not diff:
  87             return cls.CHECK
  88
  89         if diff and color:
  90             return cls.COLOR_DIFF
  91
  92         return cls.DIFF if diff else cls.YES
  93
  94
  95 # Legacy name, left for integrations.
  96 FileMode = Mode
  97
  98
  99 def read_pyproject_toml(
 100     ctx: click.Context, param: click.Parameter, value: Optional[str]
 101 ) -> Optional[str]:
 102     """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
 103
 104     Returns the path to a successfully found and read configuration file, None
 105     otherwise.
 106     """
 107     if not value:
 108         value = find_pyproject_toml(ctx.params.get("src", ()))
 109         if value is None:
 110             return None
 111
 112     try:
 113         config = parse_pyproject_toml(value)
 114     except (OSError, ValueError) as e:
 115         raise click.FileError(
 116             filename=value, hint=f"Error reading configuration file: {e}"
 117         )
 118
 119     if not config:
 120         return None
 121     else:
 122         # Sanitize the values to be Click friendly. For more information please see:
 123         # https://github.com/psf/black/issues/1458
 124         # https://github.com/pallets/click/issues/1567
 125         config = {
 126             k: str(v) if not isinstance(v, (list, dict)) else v
 127             for k, v in config.items()
 128         }
 129
 130     target_version = config.get("target_version")
 131     if target_version is not None and not isinstance(target_version, list):
 132         raise click.BadOptionUsage(
 133             "target-version", "Config key target-version must be a list"
 134         )
 135
 136     default_map: Dict[str, Any] = {}
 137     if ctx.default_map:
 138         default_map.update(ctx.default_map)
 139     default_map.update(config)
 140
 141     ctx.default_map = default_map
 142     return value
 143
 144
 145 def target_version_option_callback(
 146     c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
 147 ) -> List[TargetVersion]:
 148     """Compute the target versions from a --target-version flag.
 149
 150     This is its own function because mypy couldn't infer the type correctly
 151     when it was a lambda, causing mypyc trouble.
 152     """
 153     return [TargetVersion[val.upper()] for val in v]
 154
 155
 156 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
 157     """Compile a regular expression string in `regex`.
 158
 159     If it contains newlines, use verbose mode.
 160     """
 161     if "\n" in regex:
 162         regex = "(?x)" + regex
 163     compiled: Pattern[str] = re.compile(regex)
 164     return compiled
 165
 166
 167 def validate_regex(
 168     ctx: click.Context,
 169     param: click.Parameter,
 170     value: Optional[str],
 171 ) -> Optional[Pattern]:
 172     try:
 173         return re_compile_maybe_verbose(value) if value is not None else None
 174     except re.error:
 175         raise click.BadParameter("Not a valid regular expression")
 176
 177
 178 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 179 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
 180 @click.option(
 181     "-l",
 182     "--line-length",
 183     type=int,
 184     default=DEFAULT_LINE_LENGTH,
 185     help="How many characters per line to allow.",
 186     show_default=True,
 187 )
 188 @click.option(
 189     "-t",
 190     "--target-version",
 191     type=click.Choice([v.name.lower() for v in TargetVersion]),
 192     callback=target_version_option_callback,
 193     multiple=True,
 194     help=(
 195         "Python versions that should be supported by Black's output. [default: per-file"
 196         " auto-detection]"
 197     ),
 198 )
 199 @click.option(
 200     "--pyi",
 201     is_flag=True,
 202     help=(
 203         "Format all input files like typing stubs regardless of file extension (useful"
 204         " when piping source on standard input)."
 205     ),
 206 )
 207 @click.option(
 208     "-S",
 209     "--skip-string-normalization",
 210     is_flag=True,
 211     help="Don't normalize string quotes or prefixes.",
 212 )
 213 @click.option(
 214     "-C",
 215     "--skip-magic-trailing-comma",
 216     is_flag=True,
 217     help="Don't use trailing commas as a reason to split lines.",
 218 )
 219 @click.option(
 220     "--experimental-string-processing",
 221     is_flag=True,
 222     hidden=True,
 223     help=(
 224         "Experimental option that performs more normalization on string literals."
 225         " Currently disabled because it leads to some crashes."
 226     ),
 227 )
 228 @click.option(
 229     "--check",
 230     is_flag=True,
 231     help=(
 232         "Don't write the files back, just return the status. Return code 0 means"
 233         " nothing would change. Return code 1 means some files would be reformatted."
 234         " Return code 123 means there was an internal error."
 235     ),
 236 )
 237 @click.option(
 238     "--diff",
 239     is_flag=True,
 240     help="Don't write the files back, just output a diff for each file on stdout.",
 241 )
 242 @click.option(
 243     "--color/--no-color",
 244     is_flag=True,
 245     help="Show colored diff. Only applies when `--diff` is given.",
 246 )
 247 @click.option(
 248     "--fast/--safe",
 249     is_flag=True,
 250     help="If --fast given, skip temporary sanity checks. [default: --safe]",
 251 )
 252 @click.option(
 253     "--include",
 254     type=str,
 255     default=DEFAULT_INCLUDES,
 256     callback=validate_regex,
 257     help=(
 258         "A regular expression that matches files and directories that should be"
 259         " included on recursive searches. An empty value means all files are included"
 260         " regardless of the name. Use forward slashes for directories on all platforms"
 261         " (Windows, too). Exclusions are calculated first, inclusions later."
 262     ),
 263     show_default=True,
 264 )
 265 @click.option(
 266     "--exclude",
 267     type=str,
 268     callback=validate_regex,
 269     help=(
 270         "A regular expression that matches files and directories that should be"
 271         " excluded on recursive searches. An empty value means no paths are excluded."
 272         " Use forward slashes for directories on all platforms (Windows, too)."
 273         " Exclusions are calculated first, inclusions later. [default:"
 274         f" {DEFAULT_EXCLUDES}]"
 275     ),
 276     show_default=False,
 277 )
 278 @click.option(
 279     "--extend-exclude",
 280     type=str,
 281     callback=validate_regex,
 282     help=(
 283         "Like --exclude, but adds additional files and directories on top of the"
 284         " excluded ones. (Useful if you simply want to add to the default)"
 285     ),
 286 )
 287 @click.option(
 288     "--force-exclude",
 289     type=str,
 290     callback=validate_regex,
 291     help=(
 292         "Like --exclude, but files and directories matching this regex will be "
 293         "excluded even when they are passed explicitly as arguments."
 294     ),
 295 )
 296 @click.option(
 297     "--stdin-filename",
 298     type=str,
 299     help=(
 300         "The name of the file when passing it through stdin. Useful to make "
 301         "sure Black will respect --force-exclude option on some "
 302         "editors that rely on using stdin."
 303     ),
 304 )
 305 @click.option(
 306     "-q",
 307     "--quiet",
 308     is_flag=True,
 309     help=(
 310         "Don't emit non-error messages to stderr. Errors are still emitted; silence"
 311         " those with 2>/dev/null."
 312     ),
 313 )
 314 @click.option(
 315     "-v",
 316     "--verbose",
 317     is_flag=True,
 318     help=(
 319         "Also emit messages to stderr about files that were not changed or were ignored"
 320         " due to exclusion patterns."
 321     ),
 322 )
 323 @click.version_option(version=__version__)
 324 @click.argument(
 325     "src",
 326     nargs=-1,
 327     type=click.Path(
 328         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
 329     ),
 330     is_eager=True,
 331 )
 332 @click.option(
 333     "--config",
 334     type=click.Path(
 335         exists=True,
 336         file_okay=True,
 337         dir_okay=False,
 338         readable=True,
 339         allow_dash=False,
 340         path_type=str,
 341     ),
 342     is_eager=True,
 343     callback=read_pyproject_toml,
 344     help="Read configuration from FILE path.",
 345 )
 346 @click.pass_context
 347 def main(
 348     ctx: click.Context,
 349     code: Optional[str],
 350     line_length: int,
 351     target_version: List[TargetVersion],
 352     check: bool,
 353     diff: bool,
 354     color: bool,
 355     fast: bool,
 356     pyi: bool,
 357     skip_string_normalization: bool,
 358     skip_magic_trailing_comma: bool,
 359     experimental_string_processing: bool,
 360     quiet: bool,
 361     verbose: bool,
 362     include: Pattern,
 363     exclude: Optional[Pattern],
 364     extend_exclude: Optional[Pattern],
 365     force_exclude: Optional[Pattern],
 366     stdin_filename: Optional[str],
 367     src: Tuple[str, ...],
 368     config: Optional[str],
 369 ) -> None:
 370     """The uncompromising code formatter."""
 371     write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
 372     if target_version:
 373         versions = set(target_version)
 374     else:
 375         # We'll autodetect later.
 376         versions = set()
 377     mode = Mode(
 378         target_versions=versions,
 379         line_length=line_length,
 380         is_pyi=pyi,
 381         string_normalization=not skip_string_normalization,
 382         magic_trailing_comma=not skip_magic_trailing_comma,
 383         experimental_string_processing=experimental_string_processing,
 384     )
 385     if config and verbose:
 386         out(f"Using configuration from {config}.", bold=False, fg="blue")
 387     if code is not None:
 388         print(format_str(code, mode=mode))
 389         ctx.exit(0)
 390     report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
 391     sources = get_sources(
 392         ctx=ctx,
 393         src=src,
 394         quiet=quiet,
 395         verbose=verbose,
 396         include=include,
 397         exclude=exclude,
 398         extend_exclude=extend_exclude,
 399         force_exclude=force_exclude,
 400         report=report,
 401         stdin_filename=stdin_filename,
 402     )
 403
 404     path_empty(
 405         sources,
 406         "No Python files are present to be formatted. Nothing to do 😴",
 407         quiet,
 408         verbose,
 409         ctx,
 410     )
 411
 412     if len(sources) == 1:
 413         reformat_one(
 414             src=sources.pop(),
 415             fast=fast,
 416             write_back=write_back,
 417             mode=mode,
 418             report=report,
 419         )
 420     else:
 421         reformat_many(
 422             sources=sources, fast=fast, write_back=write_back, mode=mode, report=report
 423         )
 424
 425     if verbose or not quiet:
 426         out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
 427         click.secho(str(report), err=True)
 428     ctx.exit(report.return_code)
 429
 430
 431 def get_sources(
 432     *,
 433     ctx: click.Context,
 434     src: Tuple[str, ...],
 435     quiet: bool,
 436     verbose: bool,
 437     include: Pattern[str],
 438     exclude: Optional[Pattern[str]],
 439     extend_exclude: Optional[Pattern[str]],
 440     force_exclude: Optional[Pattern[str]],
 441     report: "Report",
 442     stdin_filename: Optional[str],
 443 ) -> Set[Path]:
 444     """Compute the set of files to be formatted."""
 445
 446     root = find_project_root(src)
 447     sources: Set[Path] = set()
 448     path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx)
 449
 450     if exclude is None:
 451         exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
 452         gitignore = get_gitignore(root)
 453     else:
 454         gitignore = None
 455
 456     for s in src:
 457         if s == "-" and stdin_filename:
 458             p = Path(stdin_filename)
 459             is_stdin = True
 460         else:
 461             p = Path(s)
 462             is_stdin = False
 463
 464         if is_stdin or p.is_file():
 465             normalized_path = normalize_path_maybe_ignore(p, root, report)
 466             if normalized_path is None:
 467                 continue
 468
 469             normalized_path = "/" + normalized_path
 470             # Hard-exclude any files that matches the `--force-exclude` regex.
 471             if force_exclude:
 472                 force_exclude_match = force_exclude.search(normalized_path)
 473             else:
 474                 force_exclude_match = None
 475             if force_exclude_match and force_exclude_match.group(0):
 476                 report.path_ignored(p, "matches the --force-exclude regular expression")
 477                 continue
 478
 479             if is_stdin:
 480                 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
 481
 482             sources.add(p)
 483         elif p.is_dir():
 484             sources.update(
 485                 gen_python_files(
 486                     p.iterdir(),
 487                     root,
 488                     include,
 489                     exclude,
 490                     extend_exclude,
 491                     force_exclude,
 492                     report,
 493                     gitignore,
 494                 )
 495             )
 496         elif s == "-":
 497             sources.add(p)
 498         else:
 499             err(f"invalid path: {s}")
 500     return sources
 501
 502
 503 def path_empty(
 504     src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
 505 ) -> None:
 506     """
 507     Exit if there is no `src` provided for formatting
 508     """
 509     if not src:
 510         if verbose or not quiet:
 511             out(msg)
 512         ctx.exit(0)
 513
 514
 515 def reformat_one(
 516     src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 517 ) -> None:
 518     """Reformat a single file under `src` without spawning child processes.
 519
 520     `fast`, `write_back`, and `mode` options are passed to
 521     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
 522     """
 523     try:
 524         changed = Changed.NO
 525
 526         if str(src) == "-":
 527             is_stdin = True
 528         elif str(src).startswith(STDIN_PLACEHOLDER):
 529             is_stdin = True
 530             # Use the original name again in case we want to print something
 531             # to the user
 532             src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
 533         else:
 534             is_stdin = False
 535
 536         if is_stdin:
 537             if src.suffix == ".pyi":
 538                 mode = replace(mode, is_pyi=True)
 539             if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
 540                 changed = Changed.YES
 541         else:
 542             cache: Cache = {}
 543             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 544                 cache = read_cache(mode)
 545                 res_src = src.resolve()
 546                 res_src_s = str(res_src)
 547                 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
 548                     changed = Changed.CACHED
 549             if changed is not Changed.CACHED and format_file_in_place(
 550                 src, fast=fast, write_back=write_back, mode=mode
 551             ):
 552                 changed = Changed.YES
 553             if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
 554                 write_back is WriteBack.CHECK and changed is Changed.NO
 555             ):
 556                 write_cache(cache, [src], mode)
 557         report.done(src, changed)
 558     except Exception as exc:
 559         if report.verbose:
 560             traceback.print_exc()
 561         report.failed(src, str(exc))
 562
 563
 564 def reformat_many(
 565     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 566 ) -> None:
 567     """Reformat multiple files using a ProcessPoolExecutor."""
 568     executor: Executor
 569     loop = asyncio.get_event_loop()
 570     worker_count = os.cpu_count()
 571     if sys.platform == "win32":
 572         # Work around https://bugs.python.org/issue26903
 573         worker_count = min(worker_count, 60)
 574     try:
 575         executor = ProcessPoolExecutor(max_workers=worker_count)
 576     except (ImportError, OSError):
 577         # we arrive here if the underlying system does not support multi-processing
 578         # like in AWS Lambda or Termux, in which case we gracefully fallback to
 579         # a ThreadPoolExecutor with just a single worker (more workers would not do us
 580         # any good due to the Global Interpreter Lock)
 581         executor = ThreadPoolExecutor(max_workers=1)
 582
 583     try:
 584         loop.run_until_complete(
 585             schedule_formatting(
 586                 sources=sources,
 587                 fast=fast,
 588                 write_back=write_back,
 589                 mode=mode,
 590                 report=report,
 591                 loop=loop,
 592                 executor=executor,
 593             )
 594         )
 595     finally:
 596         shutdown(loop)
 597         if executor is not None:
 598             executor.shutdown()
 599
 600
 601 async def schedule_formatting(
 602     sources: Set[Path],
 603     fast: bool,
 604     write_back: WriteBack,
 605     mode: Mode,
 606     report: "Report",
 607     loop: asyncio.AbstractEventLoop,
 608     executor: Executor,
 609 ) -> None:
 610     """Run formatting of `sources` in parallel using the provided `executor`.
 611
 612     (Use ProcessPoolExecutors for actual parallelism.)
 613
 614     `write_back`, `fast`, and `mode` options are passed to
 615     :func:`format_file_in_place`.
 616     """
 617     cache: Cache = {}
 618     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 619         cache = read_cache(mode)
 620         sources, cached = filter_cached(cache, sources)
 621         for src in sorted(cached):
 622             report.done(src, Changed.CACHED)
 623     if not sources:
 624         return
 625
 626     cancelled = []
 627     sources_to_cache = []
 628     lock = None
 629     if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 630         # For diff output, we need locks to ensure we don't interleave output
 631         # from different processes.
 632         manager = Manager()
 633         lock = manager.Lock()
 634     tasks = {
 635         asyncio.ensure_future(
 636             loop.run_in_executor(
 637                 executor, format_file_in_place, src, fast, mode, write_back, lock
 638             )
 639         ): src
 640         for src in sorted(sources)
 641     }
 642     pending = tasks.keys()
 643     try:
 644         loop.add_signal_handler(signal.SIGINT, cancel, pending)
 645         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
 646     except NotImplementedError:
 647         # There are no good alternatives for these on Windows.
 648         pass
 649     while pending:
 650         done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
 651         for task in done:
 652             src = tasks.pop(task)
 653             if task.cancelled():
 654                 cancelled.append(task)
 655             elif task.exception():
 656                 report.failed(src, str(task.exception()))
 657             else:
 658                 changed = Changed.YES if task.result() else Changed.NO
 659                 # If the file was written back or was successfully checked as
 660                 # well-formatted, store this information in the cache.
 661                 if write_back is WriteBack.YES or (
 662                     write_back is WriteBack.CHECK and changed is Changed.NO
 663                 ):
 664                     sources_to_cache.append(src)
 665                 report.done(src, changed)
 666     if cancelled:
 667         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
 668     if sources_to_cache:
 669         write_cache(cache, sources_to_cache, mode)
 670
 671
 672 def format_file_in_place(
 673     src: Path,
 674     fast: bool,
 675     mode: Mode,
 676     write_back: WriteBack = WriteBack.NO,
 677     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
 678 ) -> bool:
 679     """Format file under `src` path. Return True if changed.
 680
 681     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
 682     code to the file.
 683     `mode` and `fast` options are passed to :func:`format_file_contents`.
 684     """
 685     if src.suffix == ".pyi":
 686         mode = replace(mode, is_pyi=True)
 687
 688     then = datetime.utcfromtimestamp(src.stat().st_mtime)
 689     with open(src, "rb") as buf:
 690         src_contents, encoding, newline = decode_bytes(buf.read())
 691     try:
 692         dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
 693     except NothingChanged:
 694         return False
 695
 696     if write_back == WriteBack.YES:
 697         with open(src, "w", encoding=encoding, newline=newline) as f:
 698             f.write(dst_contents)
 699     elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 700         now = datetime.utcnow()
 701         src_name = f"{src}\t{then} +0000"
 702         dst_name = f"{src}\t{now} +0000"
 703         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
 704
 705         if write_back == WriteBack.COLOR_DIFF:
 706             diff_contents = color_diff(diff_contents)
 707
 708         with lock or nullcontext():
 709             f = io.TextIOWrapper(
 710                 sys.stdout.buffer,
 711                 encoding=encoding,
 712                 newline=newline,
 713                 write_through=True,
 714             )
 715             f = wrap_stream_for_windows(f)
 716             f.write(diff_contents)
 717             f.detach()
 718
 719     return True
 720
 721
 722 def format_stdin_to_stdout(
 723     fast: bool, *, write_back: WriteBack = WriteBack.NO, mode: Mode
 724 ) -> bool:
 725     """Format file on stdin. Return True if changed.
 726
 727     If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
 728     write a diff to stdout. The `mode` argument is passed to
 729     :func:`format_file_contents`.
 730     """
 731     then = datetime.utcnow()
 732     src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
 733     dst = src
 734     try:
 735         dst = format_file_contents(src, fast=fast, mode=mode)
 736         return True
 737
 738     except NothingChanged:
 739         return False
 740
 741     finally:
 742         f = io.TextIOWrapper(
 743             sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
 744         )
 745         if write_back == WriteBack.YES:
 746             f.write(dst)
 747         elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 748             now = datetime.utcnow()
 749             src_name = f"STDIN\t{then} +0000"
 750             dst_name = f"STDOUT\t{now} +0000"
 751             d = diff(src, dst, src_name, dst_name)
 752             if write_back == WriteBack.COLOR_DIFF:
 753                 d = color_diff(d)
 754                 f = wrap_stream_for_windows(f)
 755             f.write(d)
 756         f.detach()
 757
 758
 759 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
 760     """Reformat contents of a file and return new contents.
 761
 762     If `fast` is False, additionally confirm that the reformatted code is
 763     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
 764     `mode` is passed to :func:`format_str`.
 765     """
 766     if not src_contents.strip():
 767         raise NothingChanged
 768
 769     dst_contents = format_str(src_contents, mode=mode)
 770     if src_contents == dst_contents:
 771         raise NothingChanged
 772
 773     if not fast:
 774         assert_equivalent(src_contents, dst_contents)
 775
 776         # Forced second pass to work around optional trailing commas (becoming
 777         # forced trailing commas on pass 2) interacting differently with optional
 778         # parentheses.  Admittedly ugly.
 779         dst_contents_pass2 = format_str(dst_contents, mode=mode)
 780         if dst_contents != dst_contents_pass2:
 781             dst_contents = dst_contents_pass2
 782             assert_equivalent(src_contents, dst_contents, pass_num=2)
 783             assert_stable(src_contents, dst_contents, mode=mode)
 784         # Note: no need to explicitly call `assert_stable` if `dst_contents` was
 785         # the same as `dst_contents_pass2`.
 786     return dst_contents
 787
 788
 789 def format_str(src_contents: str, *, mode: Mode) -> FileContent:
 790     """Reformat a string and return new contents.
 791
 792     `mode` determines formatting options, such as how many characters per line are
 793     allowed.  Example:
 794
 795     >>> import black
 796     >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
 797     def f(arg: str = "") -> None:
 798         ...
 799
 800     A more complex example:
 801
 802     >>> print(
 803     ...   black.format_str(
 804     ...     "def f(arg:str='')->None: hey",
 805     ...     mode=black.Mode(
 806     ...       target_versions={black.TargetVersion.PY36},
 807     ...       line_length=10,
 808     ...       string_normalization=False,
 809     ...       is_pyi=False,
 810     ...     ),
 811     ...   ),
 812     ... )
 813     def f(
 814         arg: str = '',
 815     ) -> None:
 816         hey
 817
 818     """
 819     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
 820     dst_contents = []
 821     future_imports = get_future_imports(src_node)
 822     if mode.target_versions:
 823         versions = mode.target_versions
 824     else:
 825         versions = detect_target_versions(src_node)
 826     normalize_fmt_off(src_node)
 827     lines = LineGenerator(
 828         mode=mode,
 829         remove_u_prefix="unicode_literals" in future_imports
 830         or supports_feature(versions, Feature.UNICODE_LITERALS),
 831     )
 832     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
 833     empty_line = Line(mode=mode)
 834     after = 0
 835     split_line_features = {
 836         feature
 837         for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
 838         if supports_feature(versions, feature)
 839     }
 840     for current_line in lines.visit(src_node):
 841         dst_contents.append(str(empty_line) * after)
 842         before, after = elt.maybe_empty_lines(current_line)
 843         dst_contents.append(str(empty_line) * before)
 844         for line in transform_line(
 845             current_line, mode=mode, features=split_line_features
 846         ):
 847             dst_contents.append(str(line))
 848     return "".join(dst_contents)
 849
 850
 851 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
 852     """Return a tuple of (decoded_contents, encoding, newline).
 853
 854     `newline` is either CRLF or LF but `decoded_contents` is decoded with
 855     universal newlines (i.e. only contains LF).
 856     """
 857     srcbuf = io.BytesIO(src)
 858     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
 859     if not lines:
 860         return "", encoding, "\n"
 861
 862     newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
 863     srcbuf.seek(0)
 864     with io.TextIOWrapper(srcbuf, encoding) as tiow:
 865         return tiow.read(), encoding, newline
 866
 867
 868 def get_features_used(node: Node) -> Set[Feature]:
 869     """Return a set of (relatively) new Python features used in this file.
 870
 871     Currently looking for:
 872     - f-strings;
 873     - underscores in numeric literals;
 874     - trailing commas after * or ** in function signatures and calls;
 875     - positional only arguments in function signatures and lambdas;
 876     - assignment expression;
 877     - relaxed decorator syntax;
 878     """
 879     features: Set[Feature] = set()
 880     for n in node.pre_order():
 881         if n.type == token.STRING:
 882             value_head = n.value[:2]  # type: ignore
 883             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
 884                 features.add(Feature.F_STRINGS)
 885
 886         elif n.type == token.NUMBER:
 887             if "_" in n.value:  # type: ignore
 888                 features.add(Feature.NUMERIC_UNDERSCORES)
 889
 890         elif n.type == token.SLASH:
 891             if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
 892                 features.add(Feature.POS_ONLY_ARGUMENTS)
 893
 894         elif n.type == token.COLONEQUAL:
 895             features.add(Feature.ASSIGNMENT_EXPRESSIONS)
 896
 897         elif n.type == syms.decorator:
 898             if len(n.children) > 1 and not is_simple_decorator_expression(
 899                 n.children[1]
 900             ):
 901                 features.add(Feature.RELAXED_DECORATORS)
 902
 903         elif (
 904             n.type in {syms.typedargslist, syms.arglist}
 905             and n.children
 906             and n.children[-1].type == token.COMMA
 907         ):
 908             if n.type == syms.typedargslist:
 909                 feature = Feature.TRAILING_COMMA_IN_DEF
 910             else:
 911                 feature = Feature.TRAILING_COMMA_IN_CALL
 912
 913             for ch in n.children:
 914                 if ch.type in STARS:
 915                     features.add(feature)
 916
 917                 if ch.type == syms.argument:
 918                     for argch in ch.children:
 919                         if argch.type in STARS:
 920                             features.add(feature)
 921
 922     return features
 923
 924
 925 def detect_target_versions(node: Node) -> Set[TargetVersion]:
 926     """Detect the version to target based on the nodes used."""
 927     features = get_features_used(node)
 928     return {
 929         version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
 930     }
 931
 932
 933 def get_future_imports(node: Node) -> Set[str]:
 934     """Return a set of __future__ imports in the file."""
 935     imports: Set[str] = set()
 936
 937     def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
 938         for child in children:
 939             if isinstance(child, Leaf):
 940                 if child.type == token.NAME:
 941                     yield child.value
 942
 943             elif child.type == syms.import_as_name:
 944                 orig_name = child.children[0]
 945                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
 946                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
 947                 yield orig_name.value
 948
 949             elif child.type == syms.import_as_names:
 950                 yield from get_imports_from_children(child.children)
 951
 952             else:
 953                 raise AssertionError("Invalid syntax parsing imports")
 954
 955     for child in node.children:
 956         if child.type != syms.simple_stmt:
 957             break
 958
 959         first_child = child.children[0]
 960         if isinstance(first_child, Leaf):
 961             # Continue looking if we see a docstring; otherwise stop.
 962             if (
 963                 len(child.children) == 2
 964                 and first_child.type == token.STRING
 965                 and child.children[1].type == token.NEWLINE
 966             ):
 967                 continue
 968
 969             break
 970
 971         elif first_child.type == syms.import_from:
 972             module_name = first_child.children[1]
 973             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
 974                 break
 975
 976             imports |= set(get_imports_from_children(first_child.children[3:]))
 977         else:
 978             break
 979
 980     return imports
 981
 982
 983 def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None:
 984     """Raise AssertionError if `src` and `dst` aren't equivalent."""
 985     try:
 986         src_ast = parse_ast(src)
 987     except Exception as exc:
 988         raise AssertionError(
 989             "cannot use --safe with this file; failed to parse source file.  AST"
 990             f" error message: {exc}"
 991         )
 992
 993     try:
 994         dst_ast = parse_ast(dst)
 995     except Exception as exc:
 996         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
 997         raise AssertionError(
 998             f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. "
 999             "Please report a bug on https://github.com/psf/black/issues.  "
1000             f"This invalid output might be helpful: {log}"
1001         ) from None
1002
1003     src_ast_str = "\n".join(stringify_ast(src_ast))
1004     dst_ast_str = "\n".join(stringify_ast(dst_ast))
1005     if src_ast_str != dst_ast_str:
1006         log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1007         raise AssertionError(
1008             "INTERNAL ERROR: Black produced code that is not equivalent to the"
1009             f" source on pass {pass_num}.  Please report a bug on "
1010             f"https://github.com/psf/black/issues.  This diff might be helpful: {log}"
1011         ) from None
1012
1013
1014 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1015     """Raise AssertionError if `dst` reformats differently the second time."""
1016     newdst = format_str(dst, mode=mode)
1017     if dst != newdst:
1018         log = dump_to_file(
1019             str(mode),
1020             diff(src, dst, "source", "first pass"),
1021             diff(dst, newdst, "first pass", "second pass"),
1022         )
1023         raise AssertionError(
1024             "INTERNAL ERROR: Black produced different code on the second pass of the"
1025             " formatter.  Please report a bug on https://github.com/psf/black/issues."
1026             f"  This diff might be helpful: {log}"
1027         ) from None
1028
1029
1030 @contextmanager
1031 def nullcontext() -> Iterator[None]:
1032     """Return an empty context manager.
1033
1034     To be used like `nullcontext` in Python 3.7.
1035     """
1036     yield
1037
1038
1039 def patch_click() -> None:
1040     """Make Click not crash on Python 3.6 with LANG=C.
1041
1042     On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1043     default which restricts paths that it can access during the lifetime of the
1044     application.  Click refuses to work in this scenario by raising a RuntimeError.
1045
1046     In case of Black the likelihood that non-ASCII characters are going to be used in
1047     file paths is minimal since it's Python source code.  Moreover, this crash was
1048     spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1049     """
1050     try:
1051         from click import core
1052         from click import _unicodefun  # type: ignore
1053     except ModuleNotFoundError:
1054         return
1055
1056     for module in (core, _unicodefun):
1057         if hasattr(module, "_verify_python3_env"):
1058             module._verify_python3_env = lambda: None  # type: ignore
1059         if hasattr(module, "_verify_python_env"):
1060             module._verify_python_env = lambda: None  # type: ignore
1061
1062
1063 def patched_main() -> None:
1064     freeze_support()
1065     patch_click()
1066     main()
1067
1068
1069 if __name__ == "__main__":
1070     patched_main()