src/black/__init__.py

   1 import asyncio
   2 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
   3 from contextlib import contextmanager
   4 from datetime import datetime
   5 from enum import Enum
   6 import io
   7 from multiprocessing import Manager, freeze_support
   8 import os
   9 from pathlib import Path
  10 import regex as re
  11 import signal
  12 import sys
  13 import tokenize
  14 import traceback
  15 from typing import (
  16     Any,
  17     Dict,
  18     Generator,
  19     Iterator,
  20     List,
  21     Optional,
  22     Pattern,
  23     Set,
  24     Sized,
  25     Tuple,
  26     Union,
  27 )
  28
  29 from dataclasses import replace
  30 import click
  31
  32 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
  33 from black.const import STDIN_PLACEHOLDER
  34 from black.nodes import STARS, syms, is_simple_decorator_expression
  35 from black.lines import Line, EmptyLineTracker
  36 from black.linegen import transform_line, LineGenerator, LN
  37 from black.comments import normalize_fmt_off
  38 from black.mode import Mode, TargetVersion
  39 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
  40 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
  41 from black.concurrency import cancel, shutdown, maybe_install_uvloop
  42 from black.output import dump_to_file, diff, color_diff, out, err
  43 from black.report import Report, Changed
  44 from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml
  45 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
  46 from black.files import wrap_stream_for_windows
  47 from black.parsing import InvalidInput  # noqa F401
  48 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
  49
  50
  51 # lib2to3 fork
  52 from blib2to3.pytree import Node, Leaf
  53 from blib2to3.pgen2 import token
  54
  55 from _black_version import version as __version__
  56
  57 # types
  58 FileContent = str
  59 Encoding = str
  60 NewLine = str
  61
  62
  63 class NothingChanged(UserWarning):
  64     """Raised when reformatted code is the same as source."""
  65
  66
  67 class WriteBack(Enum):
  68     NO = 0
  69     YES = 1
  70     DIFF = 2
  71     CHECK = 3
  72     COLOR_DIFF = 4
  73
  74     @classmethod
  75     def from_configuration(
  76         cls, *, check: bool, diff: bool, color: bool = False
  77     ) -> "WriteBack":
  78         if check and not diff:
  79             return cls.CHECK
  80
  81         if diff and color:
  82             return cls.COLOR_DIFF
  83
  84         return cls.DIFF if diff else cls.YES
  85
  86
  87 # Legacy name, left for integrations.
  88 FileMode = Mode
  89
  90
  91 def read_pyproject_toml(
  92     ctx: click.Context, param: click.Parameter, value: Optional[str]
  93 ) -> Optional[str]:
  94     """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
  95
  96     Returns the path to a successfully found and read configuration file, None
  97     otherwise.
  98     """
  99     if not value:
 100         value = find_pyproject_toml(ctx.params.get("src", ()))
 101         if value is None:
 102             return None
 103
 104     try:
 105         config = parse_pyproject_toml(value)
 106     except (OSError, ValueError) as e:
 107         raise click.FileError(
 108             filename=value, hint=f"Error reading configuration file: {e}"
 109         )
 110
 111     if not config:
 112         return None
 113     else:
 114         # Sanitize the values to be Click friendly. For more information please see:
 115         # https://github.com/psf/black/issues/1458
 116         # https://github.com/pallets/click/issues/1567
 117         config = {
 118             k: str(v) if not isinstance(v, (list, dict)) else v
 119             for k, v in config.items()
 120         }
 121
 122     target_version = config.get("target_version")
 123     if target_version is not None and not isinstance(target_version, list):
 124         raise click.BadOptionUsage(
 125             "target-version", "Config key target-version must be a list"
 126         )
 127
 128     default_map: Dict[str, Any] = {}
 129     if ctx.default_map:
 130         default_map.update(ctx.default_map)
 131     default_map.update(config)
 132
 133     ctx.default_map = default_map
 134     return value
 135
 136
 137 def target_version_option_callback(
 138     c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
 139 ) -> List[TargetVersion]:
 140     """Compute the target versions from a --target-version flag.
 141
 142     This is its own function because mypy couldn't infer the type correctly
 143     when it was a lambda, causing mypyc trouble.
 144     """
 145     return [TargetVersion[val.upper()] for val in v]
 146
 147
 148 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
 149     """Compile a regular expression string in `regex`.
 150
 151     If it contains newlines, use verbose mode.
 152     """
 153     if "\n" in regex:
 154         regex = "(?x)" + regex
 155     compiled: Pattern[str] = re.compile(regex)
 156     return compiled
 157
 158
 159 def validate_regex(
 160     ctx: click.Context,
 161     param: click.Parameter,
 162     value: Optional[str],
 163 ) -> Optional[Pattern]:
 164     try:
 165         return re_compile_maybe_verbose(value) if value is not None else None
 166     except re.error:
 167         raise click.BadParameter("Not a valid regular expression")
 168
 169
 170 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 171 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
 172 @click.option(
 173     "-l",
 174     "--line-length",
 175     type=int,
 176     default=DEFAULT_LINE_LENGTH,
 177     help="How many characters per line to allow.",
 178     show_default=True,
 179 )
 180 @click.option(
 181     "-t",
 182     "--target-version",
 183     type=click.Choice([v.name.lower() for v in TargetVersion]),
 184     callback=target_version_option_callback,
 185     multiple=True,
 186     help=(
 187         "Python versions that should be supported by Black's output. [default: per-file"
 188         " auto-detection]"
 189     ),
 190 )
 191 @click.option(
 192     "--pyi",
 193     is_flag=True,
 194     help=(
 195         "Format all input files like typing stubs regardless of file extension (useful"
 196         " when piping source on standard input)."
 197     ),
 198 )
 199 @click.option(
 200     "-S",
 201     "--skip-string-normalization",
 202     is_flag=True,
 203     help="Don't normalize string quotes or prefixes.",
 204 )
 205 @click.option(
 206     "-C",
 207     "--skip-magic-trailing-comma",
 208     is_flag=True,
 209     help="Don't use trailing commas as a reason to split lines.",
 210 )
 211 @click.option(
 212     "--experimental-string-processing",
 213     is_flag=True,
 214     hidden=True,
 215     help=(
 216         "Experimental option that performs more normalization on string literals."
 217         " Currently disabled because it leads to some crashes."
 218     ),
 219 )
 220 @click.option(
 221     "--check",
 222     is_flag=True,
 223     help=(
 224         "Don't write the files back, just return the status. Return code 0 means"
 225         " nothing would change. Return code 1 means some files would be reformatted."
 226         " Return code 123 means there was an internal error."
 227     ),
 228 )
 229 @click.option(
 230     "--diff",
 231     is_flag=True,
 232     help="Don't write the files back, just output a diff for each file on stdout.",
 233 )
 234 @click.option(
 235     "--color/--no-color",
 236     is_flag=True,
 237     help="Show colored diff. Only applies when `--diff` is given.",
 238 )
 239 @click.option(
 240     "--fast/--safe",
 241     is_flag=True,
 242     help="If --fast given, skip temporary sanity checks. [default: --safe]",
 243 )
 244 @click.option(
 245     "--include",
 246     type=str,
 247     default=DEFAULT_INCLUDES,
 248     callback=validate_regex,
 249     help=(
 250         "A regular expression that matches files and directories that should be"
 251         " included on recursive searches. An empty value means all files are included"
 252         " regardless of the name. Use forward slashes for directories on all platforms"
 253         " (Windows, too). Exclusions are calculated first, inclusions later."
 254     ),
 255     show_default=True,
 256 )
 257 @click.option(
 258     "--exclude",
 259     type=str,
 260     callback=validate_regex,
 261     help=(
 262         "A regular expression that matches files and directories that should be"
 263         " excluded on recursive searches. An empty value means no paths are excluded."
 264         " Use forward slashes for directories on all platforms (Windows, too)."
 265         " Exclusions are calculated first, inclusions later. [default:"
 266         f" {DEFAULT_EXCLUDES}]"
 267     ),
 268     show_default=False,
 269 )
 270 @click.option(
 271     "--extend-exclude",
 272     type=str,
 273     callback=validate_regex,
 274     help=(
 275         "Like --exclude, but adds additional files and directories on top of the"
 276         " excluded ones. (Useful if you simply want to add to the default)"
 277     ),
 278 )
 279 @click.option(
 280     "--force-exclude",
 281     type=str,
 282     callback=validate_regex,
 283     help=(
 284         "Like --exclude, but files and directories matching this regex will be "
 285         "excluded even when they are passed explicitly as arguments."
 286     ),
 287 )
 288 @click.option(
 289     "--stdin-filename",
 290     type=str,
 291     help=(
 292         "The name of the file when passing it through stdin. Useful to make "
 293         "sure Black will respect --force-exclude option on some "
 294         "editors that rely on using stdin."
 295     ),
 296 )
 297 @click.option(
 298     "-q",
 299     "--quiet",
 300     is_flag=True,
 301     help=(
 302         "Don't emit non-error messages to stderr. Errors are still emitted; silence"
 303         " those with 2>/dev/null."
 304     ),
 305 )
 306 @click.option(
 307     "-v",
 308     "--verbose",
 309     is_flag=True,
 310     help=(
 311         "Also emit messages to stderr about files that were not changed or were ignored"
 312         " due to exclusion patterns."
 313     ),
 314 )
 315 @click.version_option(version=__version__)
 316 @click.argument(
 317     "src",
 318     nargs=-1,
 319     type=click.Path(
 320         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
 321     ),
 322     is_eager=True,
 323 )
 324 @click.option(
 325     "--config",
 326     type=click.Path(
 327         exists=True,
 328         file_okay=True,
 329         dir_okay=False,
 330         readable=True,
 331         allow_dash=False,
 332         path_type=str,
 333     ),
 334     is_eager=True,
 335     callback=read_pyproject_toml,
 336     help="Read configuration from FILE path.",
 337 )
 338 @click.pass_context
 339 def main(
 340     ctx: click.Context,
 341     code: Optional[str],
 342     line_length: int,
 343     target_version: List[TargetVersion],
 344     check: bool,
 345     diff: bool,
 346     color: bool,
 347     fast: bool,
 348     pyi: bool,
 349     skip_string_normalization: bool,
 350     skip_magic_trailing_comma: bool,
 351     experimental_string_processing: bool,
 352     quiet: bool,
 353     verbose: bool,
 354     include: Pattern,
 355     exclude: Optional[Pattern],
 356     extend_exclude: Optional[Pattern],
 357     force_exclude: Optional[Pattern],
 358     stdin_filename: Optional[str],
 359     src: Tuple[str, ...],
 360     config: Optional[str],
 361 ) -> None:
 362     """The uncompromising code formatter."""
 363     write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
 364     if target_version:
 365         versions = set(target_version)
 366     else:
 367         # We'll autodetect later.
 368         versions = set()
 369     mode = Mode(
 370         target_versions=versions,
 371         line_length=line_length,
 372         is_pyi=pyi,
 373         string_normalization=not skip_string_normalization,
 374         magic_trailing_comma=not skip_magic_trailing_comma,
 375         experimental_string_processing=experimental_string_processing,
 376     )
 377     if config and verbose:
 378         out(f"Using configuration from {config}.", bold=False, fg="blue")
 379
 380     if code is not None:
 381         # Run in quiet mode by default with -c; the extra output isn't useful.
 382         # You can still pass -v to get verbose output.
 383         quiet = True
 384
 385     report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
 386
 387     if code is not None:
 388         reformat_code(
 389             content=code, fast=fast, write_back=write_back, mode=mode, report=report
 390         )
 391     else:
 392         sources = get_sources(
 393             ctx=ctx,
 394             src=src,
 395             quiet=quiet,
 396             verbose=verbose,
 397             include=include,
 398             exclude=exclude,
 399             extend_exclude=extend_exclude,
 400             force_exclude=force_exclude,
 401             report=report,
 402             stdin_filename=stdin_filename,
 403         )
 404
 405         path_empty(
 406             sources,
 407             "No Python files are present to be formatted. Nothing to do 😴",
 408             quiet,
 409             verbose,
 410             ctx,
 411         )
 412
 413         if len(sources) == 1:
 414             reformat_one(
 415                 src=sources.pop(),
 416                 fast=fast,
 417                 write_back=write_back,
 418                 mode=mode,
 419                 report=report,
 420             )
 421         else:
 422             reformat_many(
 423                 sources=sources,
 424                 fast=fast,
 425                 write_back=write_back,
 426                 mode=mode,
 427                 report=report,
 428             )
 429
 430     if verbose or not quiet:
 431         out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
 432         if code is None:
 433             click.secho(str(report), err=True)
 434     ctx.exit(report.return_code)
 435
 436
 437 def get_sources(
 438     *,
 439     ctx: click.Context,
 440     src: Tuple[str, ...],
 441     quiet: bool,
 442     verbose: bool,
 443     include: Pattern[str],
 444     exclude: Optional[Pattern[str]],
 445     extend_exclude: Optional[Pattern[str]],
 446     force_exclude: Optional[Pattern[str]],
 447     report: "Report",
 448     stdin_filename: Optional[str],
 449 ) -> Set[Path]:
 450     """Compute the set of files to be formatted."""
 451
 452     root = find_project_root(src)
 453     sources: Set[Path] = set()
 454     path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx)
 455
 456     if exclude is None:
 457         exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
 458         gitignore = get_gitignore(root)
 459     else:
 460         gitignore = None
 461
 462     for s in src:
 463         if s == "-" and stdin_filename:
 464             p = Path(stdin_filename)
 465             is_stdin = True
 466         else:
 467             p = Path(s)
 468             is_stdin = False
 469
 470         if is_stdin or p.is_file():
 471             normalized_path = normalize_path_maybe_ignore(p, root, report)
 472             if normalized_path is None:
 473                 continue
 474
 475             normalized_path = "/" + normalized_path
 476             # Hard-exclude any files that matches the `--force-exclude` regex.
 477             if force_exclude:
 478                 force_exclude_match = force_exclude.search(normalized_path)
 479             else:
 480                 force_exclude_match = None
 481             if force_exclude_match and force_exclude_match.group(0):
 482                 report.path_ignored(p, "matches the --force-exclude regular expression")
 483                 continue
 484
 485             if is_stdin:
 486                 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
 487
 488             sources.add(p)
 489         elif p.is_dir():
 490             sources.update(
 491                 gen_python_files(
 492                     p.iterdir(),
 493                     root,
 494                     include,
 495                     exclude,
 496                     extend_exclude,
 497                     force_exclude,
 498                     report,
 499                     gitignore,
 500                 )
 501             )
 502         elif s == "-":
 503             sources.add(p)
 504         else:
 505             err(f"invalid path: {s}")
 506     return sources
 507
 508
 509 def path_empty(
 510     src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
 511 ) -> None:
 512     """
 513     Exit if there is no `src` provided for formatting
 514     """
 515     if not src:
 516         if verbose or not quiet:
 517             out(msg)
 518         ctx.exit(0)
 519
 520
 521 def reformat_code(
 522     content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report
 523 ) -> None:
 524     """
 525     Reformat and print out `content` without spawning child processes.
 526     Similar to `reformat_one`, but for string content.
 527
 528     `fast`, `write_back`, and `mode` options are passed to
 529     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
 530     """
 531     path = Path("<string>")
 532     try:
 533         changed = Changed.NO
 534         if format_stdin_to_stdout(
 535             content=content, fast=fast, write_back=write_back, mode=mode
 536         ):
 537             changed = Changed.YES
 538         report.done(path, changed)
 539     except Exception as exc:
 540         if report.verbose:
 541             traceback.print_exc()
 542         report.failed(path, str(exc))
 543
 544
 545 def reformat_one(
 546     src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 547 ) -> None:
 548     """Reformat a single file under `src` without spawning child processes.
 549
 550     `fast`, `write_back`, and `mode` options are passed to
 551     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
 552     """
 553     try:
 554         changed = Changed.NO
 555
 556         if str(src) == "-":
 557             is_stdin = True
 558         elif str(src).startswith(STDIN_PLACEHOLDER):
 559             is_stdin = True
 560             # Use the original name again in case we want to print something
 561             # to the user
 562             src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
 563         else:
 564             is_stdin = False
 565
 566         if is_stdin:
 567             if src.suffix == ".pyi":
 568                 mode = replace(mode, is_pyi=True)
 569             if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
 570                 changed = Changed.YES
 571         else:
 572             cache: Cache = {}
 573             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 574                 cache = read_cache(mode)
 575                 res_src = src.resolve()
 576                 res_src_s = str(res_src)
 577                 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
 578                     changed = Changed.CACHED
 579             if changed is not Changed.CACHED and format_file_in_place(
 580                 src, fast=fast, write_back=write_back, mode=mode
 581             ):
 582                 changed = Changed.YES
 583             if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
 584                 write_back is WriteBack.CHECK and changed is Changed.NO
 585             ):
 586                 write_cache(cache, [src], mode)
 587         report.done(src, changed)
 588     except Exception as exc:
 589         if report.verbose:
 590             traceback.print_exc()
 591         report.failed(src, str(exc))
 592
 593
 594 def reformat_many(
 595     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 596 ) -> None:
 597     """Reformat multiple files using a ProcessPoolExecutor."""
 598     executor: Executor
 599     loop = asyncio.get_event_loop()
 600     worker_count = os.cpu_count()
 601     if sys.platform == "win32":
 602         # Work around https://bugs.python.org/issue26903
 603         worker_count = min(worker_count, 60)
 604     try:
 605         executor = ProcessPoolExecutor(max_workers=worker_count)
 606     except (ImportError, OSError):
 607         # we arrive here if the underlying system does not support multi-processing
 608         # like in AWS Lambda or Termux, in which case we gracefully fallback to
 609         # a ThreadPoolExecutor with just a single worker (more workers would not do us
 610         # any good due to the Global Interpreter Lock)
 611         executor = ThreadPoolExecutor(max_workers=1)
 612
 613     try:
 614         loop.run_until_complete(
 615             schedule_formatting(
 616                 sources=sources,
 617                 fast=fast,
 618                 write_back=write_back,
 619                 mode=mode,
 620                 report=report,
 621                 loop=loop,
 622                 executor=executor,
 623             )
 624         )
 625     finally:
 626         shutdown(loop)
 627         if executor is not None:
 628             executor.shutdown()
 629
 630
 631 async def schedule_formatting(
 632     sources: Set[Path],
 633     fast: bool,
 634     write_back: WriteBack,
 635     mode: Mode,
 636     report: "Report",
 637     loop: asyncio.AbstractEventLoop,
 638     executor: Executor,
 639 ) -> None:
 640     """Run formatting of `sources` in parallel using the provided `executor`.
 641
 642     (Use ProcessPoolExecutors for actual parallelism.)
 643
 644     `write_back`, `fast`, and `mode` options are passed to
 645     :func:`format_file_in_place`.
 646     """
 647     cache: Cache = {}
 648     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 649         cache = read_cache(mode)
 650         sources, cached = filter_cached(cache, sources)
 651         for src in sorted(cached):
 652             report.done(src, Changed.CACHED)
 653     if not sources:
 654         return
 655
 656     cancelled = []
 657     sources_to_cache = []
 658     lock = None
 659     if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 660         # For diff output, we need locks to ensure we don't interleave output
 661         # from different processes.
 662         manager = Manager()
 663         lock = manager.Lock()
 664     tasks = {
 665         asyncio.ensure_future(
 666             loop.run_in_executor(
 667                 executor, format_file_in_place, src, fast, mode, write_back, lock
 668             )
 669         ): src
 670         for src in sorted(sources)
 671     }
 672     pending = tasks.keys()
 673     try:
 674         loop.add_signal_handler(signal.SIGINT, cancel, pending)
 675         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
 676     except NotImplementedError:
 677         # There are no good alternatives for these on Windows.
 678         pass
 679     while pending:
 680         done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
 681         for task in done:
 682             src = tasks.pop(task)
 683             if task.cancelled():
 684                 cancelled.append(task)
 685             elif task.exception():
 686                 report.failed(src, str(task.exception()))
 687             else:
 688                 changed = Changed.YES if task.result() else Changed.NO
 689                 # If the file was written back or was successfully checked as
 690                 # well-formatted, store this information in the cache.
 691                 if write_back is WriteBack.YES or (
 692                     write_back is WriteBack.CHECK and changed is Changed.NO
 693                 ):
 694                     sources_to_cache.append(src)
 695                 report.done(src, changed)
 696     if cancelled:
 697         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
 698     if sources_to_cache:
 699         write_cache(cache, sources_to_cache, mode)
 700
 701
 702 def format_file_in_place(
 703     src: Path,
 704     fast: bool,
 705     mode: Mode,
 706     write_back: WriteBack = WriteBack.NO,
 707     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
 708 ) -> bool:
 709     """Format file under `src` path. Return True if changed.
 710
 711     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
 712     code to the file.
 713     `mode` and `fast` options are passed to :func:`format_file_contents`.
 714     """
 715     if src.suffix == ".pyi":
 716         mode = replace(mode, is_pyi=True)
 717
 718     then = datetime.utcfromtimestamp(src.stat().st_mtime)
 719     with open(src, "rb") as buf:
 720         src_contents, encoding, newline = decode_bytes(buf.read())
 721     try:
 722         dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
 723     except NothingChanged:
 724         return False
 725
 726     if write_back == WriteBack.YES:
 727         with open(src, "w", encoding=encoding, newline=newline) as f:
 728             f.write(dst_contents)
 729     elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 730         now = datetime.utcnow()
 731         src_name = f"{src}\t{then} +0000"
 732         dst_name = f"{src}\t{now} +0000"
 733         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
 734
 735         if write_back == WriteBack.COLOR_DIFF:
 736             diff_contents = color_diff(diff_contents)
 737
 738         with lock or nullcontext():
 739             f = io.TextIOWrapper(
 740                 sys.stdout.buffer,
 741                 encoding=encoding,
 742                 newline=newline,
 743                 write_through=True,
 744             )
 745             f = wrap_stream_for_windows(f)
 746             f.write(diff_contents)
 747             f.detach()
 748
 749     return True
 750
 751
 752 def format_stdin_to_stdout(
 753     fast: bool,
 754     *,
 755     content: Optional[str] = None,
 756     write_back: WriteBack = WriteBack.NO,
 757     mode: Mode,
 758 ) -> bool:
 759     """Format file on stdin. Return True if changed.
 760
 761     If content is None, it's read from sys.stdin.
 762
 763     If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
 764     write a diff to stdout. The `mode` argument is passed to
 765     :func:`format_file_contents`.
 766     """
 767     then = datetime.utcnow()
 768
 769     if content is None:
 770         src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
 771     else:
 772         src, encoding, newline = content, "utf-8", ""
 773
 774     dst = src
 775     try:
 776         dst = format_file_contents(src, fast=fast, mode=mode)
 777         return True
 778
 779     except NothingChanged:
 780         return False
 781
 782     finally:
 783         f = io.TextIOWrapper(
 784             sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
 785         )
 786         if write_back == WriteBack.YES:
 787             # Make sure there's a newline after the content
 788             dst += "" if dst[-1] == "\n" else "\n"
 789             f.write(dst)
 790         elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 791             now = datetime.utcnow()
 792             src_name = f"STDIN\t{then} +0000"
 793             dst_name = f"STDOUT\t{now} +0000"
 794             d = diff(src, dst, src_name, dst_name)
 795             if write_back == WriteBack.COLOR_DIFF:
 796                 d = color_diff(d)
 797                 f = wrap_stream_for_windows(f)
 798             f.write(d)
 799         f.detach()
 800
 801
 802 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
 803     """Reformat contents of a file and return new contents.
 804
 805     If `fast` is False, additionally confirm that the reformatted code is
 806     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
 807     `mode` is passed to :func:`format_str`.
 808     """
 809     if not src_contents.strip():
 810         raise NothingChanged
 811
 812     dst_contents = format_str(src_contents, mode=mode)
 813     if src_contents == dst_contents:
 814         raise NothingChanged
 815
 816     if not fast:
 817         assert_equivalent(src_contents, dst_contents)
 818
 819         # Forced second pass to work around optional trailing commas (becoming
 820         # forced trailing commas on pass 2) interacting differently with optional
 821         # parentheses.  Admittedly ugly.
 822         dst_contents_pass2 = format_str(dst_contents, mode=mode)
 823         if dst_contents != dst_contents_pass2:
 824             dst_contents = dst_contents_pass2
 825             assert_equivalent(src_contents, dst_contents, pass_num=2)
 826             assert_stable(src_contents, dst_contents, mode=mode)
 827         # Note: no need to explicitly call `assert_stable` if `dst_contents` was
 828         # the same as `dst_contents_pass2`.
 829     return dst_contents
 830
 831
 832 def format_str(src_contents: str, *, mode: Mode) -> FileContent:
 833     """Reformat a string and return new contents.
 834
 835     `mode` determines formatting options, such as how many characters per line are
 836     allowed.  Example:
 837
 838     >>> import black
 839     >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
 840     def f(arg: str = "") -> None:
 841         ...
 842
 843     A more complex example:
 844
 845     >>> print(
 846     ...   black.format_str(
 847     ...     "def f(arg:str='')->None: hey",
 848     ...     mode=black.Mode(
 849     ...       target_versions={black.TargetVersion.PY36},
 850     ...       line_length=10,
 851     ...       string_normalization=False,
 852     ...       is_pyi=False,
 853     ...     ),
 854     ...   ),
 855     ... )
 856     def f(
 857         arg: str = '',
 858     ) -> None:
 859         hey
 860
 861     """
 862     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
 863     dst_contents = []
 864     future_imports = get_future_imports(src_node)
 865     if mode.target_versions:
 866         versions = mode.target_versions
 867     else:
 868         versions = detect_target_versions(src_node)
 869     normalize_fmt_off(src_node)
 870     lines = LineGenerator(
 871         mode=mode,
 872         remove_u_prefix="unicode_literals" in future_imports
 873         or supports_feature(versions, Feature.UNICODE_LITERALS),
 874     )
 875     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
 876     empty_line = Line(mode=mode)
 877     after = 0
 878     split_line_features = {
 879         feature
 880         for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
 881         if supports_feature(versions, feature)
 882     }
 883     for current_line in lines.visit(src_node):
 884         dst_contents.append(str(empty_line) * after)
 885         before, after = elt.maybe_empty_lines(current_line)
 886         dst_contents.append(str(empty_line) * before)
 887         for line in transform_line(
 888             current_line, mode=mode, features=split_line_features
 889         ):
 890             dst_contents.append(str(line))
 891     return "".join(dst_contents)
 892
 893
 894 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
 895     """Return a tuple of (decoded_contents, encoding, newline).
 896
 897     `newline` is either CRLF or LF but `decoded_contents` is decoded with
 898     universal newlines (i.e. only contains LF).
 899     """
 900     srcbuf = io.BytesIO(src)
 901     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
 902     if not lines:
 903         return "", encoding, "\n"
 904
 905     newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
 906     srcbuf.seek(0)
 907     with io.TextIOWrapper(srcbuf, encoding) as tiow:
 908         return tiow.read(), encoding, newline
 909
 910
 911 def get_features_used(node: Node) -> Set[Feature]:
 912     """Return a set of (relatively) new Python features used in this file.
 913
 914     Currently looking for:
 915     - f-strings;
 916     - underscores in numeric literals;
 917     - trailing commas after * or ** in function signatures and calls;
 918     - positional only arguments in function signatures and lambdas;
 919     - assignment expression;
 920     - relaxed decorator syntax;
 921     """
 922     features: Set[Feature] = set()
 923     for n in node.pre_order():
 924         if n.type == token.STRING:
 925             value_head = n.value[:2]  # type: ignore
 926             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
 927                 features.add(Feature.F_STRINGS)
 928
 929         elif n.type == token.NUMBER:
 930             if "_" in n.value:  # type: ignore
 931                 features.add(Feature.NUMERIC_UNDERSCORES)
 932
 933         elif n.type == token.SLASH:
 934             if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
 935                 features.add(Feature.POS_ONLY_ARGUMENTS)
 936
 937         elif n.type == token.COLONEQUAL:
 938             features.add(Feature.ASSIGNMENT_EXPRESSIONS)
 939
 940         elif n.type == syms.decorator:
 941             if len(n.children) > 1 and not is_simple_decorator_expression(
 942                 n.children[1]
 943             ):
 944                 features.add(Feature.RELAXED_DECORATORS)
 945
 946         elif (
 947             n.type in {syms.typedargslist, syms.arglist}
 948             and n.children
 949             and n.children[-1].type == token.COMMA
 950         ):
 951             if n.type == syms.typedargslist:
 952                 feature = Feature.TRAILING_COMMA_IN_DEF
 953             else:
 954                 feature = Feature.TRAILING_COMMA_IN_CALL
 955
 956             for ch in n.children:
 957                 if ch.type in STARS:
 958                     features.add(feature)
 959
 960                 if ch.type == syms.argument:
 961                     for argch in ch.children:
 962                         if argch.type in STARS:
 963                             features.add(feature)
 964
 965     return features
 966
 967
 968 def detect_target_versions(node: Node) -> Set[TargetVersion]:
 969     """Detect the version to target based on the nodes used."""
 970     features = get_features_used(node)
 971     return {
 972         version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
 973     }
 974
 975
 976 def get_future_imports(node: Node) -> Set[str]:
 977     """Return a set of __future__ imports in the file."""
 978     imports: Set[str] = set()
 979
 980     def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
 981         for child in children:
 982             if isinstance(child, Leaf):
 983                 if child.type == token.NAME:
 984                     yield child.value
 985
 986             elif child.type == syms.import_as_name:
 987                 orig_name = child.children[0]
 988                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
 989                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
 990                 yield orig_name.value
 991
 992             elif child.type == syms.import_as_names:
 993                 yield from get_imports_from_children(child.children)
 994
 995             else:
 996                 raise AssertionError("Invalid syntax parsing imports")
 997
 998     for child in node.children:
 999         if child.type != syms.simple_stmt:
1000             break
1001
1002         first_child = child.children[0]
1003         if isinstance(first_child, Leaf):
1004             # Continue looking if we see a docstring; otherwise stop.
1005             if (
1006                 len(child.children) == 2
1007                 and first_child.type == token.STRING
1008                 and child.children[1].type == token.NEWLINE
1009             ):
1010                 continue
1011
1012             break
1013
1014         elif first_child.type == syms.import_from:
1015             module_name = first_child.children[1]
1016             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
1017                 break
1018
1019             imports |= set(get_imports_from_children(first_child.children[3:]))
1020         else:
1021             break
1022
1023     return imports
1024
1025
1026 def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None:
1027     """Raise AssertionError if `src` and `dst` aren't equivalent."""
1028     try:
1029         src_ast = parse_ast(src)
1030     except Exception as exc:
1031         raise AssertionError(
1032             "cannot use --safe with this file; failed to parse source file.  AST"
1033             f" error message: {exc}"
1034         )
1035
1036     try:
1037         dst_ast = parse_ast(dst)
1038     except Exception as exc:
1039         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
1040         raise AssertionError(
1041             f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. "
1042             "Please report a bug on https://github.com/psf/black/issues.  "
1043             f"This invalid output might be helpful: {log}"
1044         ) from None
1045
1046     src_ast_str = "\n".join(stringify_ast(src_ast))
1047     dst_ast_str = "\n".join(stringify_ast(dst_ast))
1048     if src_ast_str != dst_ast_str:
1049         log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1050         raise AssertionError(
1051             "INTERNAL ERROR: Black produced code that is not equivalent to the"
1052             f" source on pass {pass_num}.  Please report a bug on "
1053             f"https://github.com/psf/black/issues.  This diff might be helpful: {log}"
1054         ) from None
1055
1056
1057 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1058     """Raise AssertionError if `dst` reformats differently the second time."""
1059     newdst = format_str(dst, mode=mode)
1060     if dst != newdst:
1061         log = dump_to_file(
1062             str(mode),
1063             diff(src, dst, "source", "first pass"),
1064             diff(dst, newdst, "first pass", "second pass"),
1065         )
1066         raise AssertionError(
1067             "INTERNAL ERROR: Black produced different code on the second pass of the"
1068             " formatter.  Please report a bug on https://github.com/psf/black/issues."
1069             f"  This diff might be helpful: {log}"
1070         ) from None
1071
1072
1073 @contextmanager
1074 def nullcontext() -> Iterator[None]:
1075     """Return an empty context manager.
1076
1077     To be used like `nullcontext` in Python 3.7.
1078     """
1079     yield
1080
1081
1082 def patch_click() -> None:
1083     """Make Click not crash on Python 3.6 with LANG=C.
1084
1085     On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1086     default which restricts paths that it can access during the lifetime of the
1087     application.  Click refuses to work in this scenario by raising a RuntimeError.
1088
1089     In case of Black the likelihood that non-ASCII characters are going to be used in
1090     file paths is minimal since it's Python source code.  Moreover, this crash was
1091     spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1092     """
1093     try:
1094         from click import core
1095         from click import _unicodefun  # type: ignore
1096     except ModuleNotFoundError:
1097         return
1098
1099     for module in (core, _unicodefun):
1100         if hasattr(module, "_verify_python3_env"):
1101             module._verify_python3_env = lambda: None  # type: ignore
1102         if hasattr(module, "_verify_python_env"):
1103             module._verify_python_env = lambda: None  # type: ignore
1104
1105
1106 def patched_main() -> None:
1107     maybe_install_uvloop()
1108     freeze_support()
1109     patch_click()
1110     main()
1111
1112
1113 if __name__ == "__main__":
1114     patched_main()