src/black/__init__.py

   1 import asyncio
   2 from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
   3 from contextlib import contextmanager
   4 from datetime import datetime
   5 from enum import Enum
   6 import io
   7 from multiprocessing import Manager, freeze_support
   8 import os
   9 from pathlib import Path
  10 import regex as re
  11 import signal
  12 import sys
  13 import tokenize
  14 import traceback
  15 from typing import (
  16     Any,
  17     Dict,
  18     Generator,
  19     Iterator,
  20     List,
  21     Optional,
  22     Pattern,
  23     Set,
  24     Sized,
  25     Tuple,
  26     Union,
  27 )
  28
  29 from dataclasses import replace
  30 import click
  31
  32 from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES
  33 from black.const import STDIN_PLACEHOLDER
  34 from black.nodes import STARS, syms, is_simple_decorator_expression
  35 from black.lines import Line, EmptyLineTracker
  36 from black.linegen import transform_line, LineGenerator, LN
  37 from black.comments import normalize_fmt_off
  38 from black.mode import Mode, TargetVersion
  39 from black.mode import Feature, supports_feature, VERSION_TO_FEATURES
  40 from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache
  41 from black.concurrency import cancel, shutdown
  42 from black.output import dump_to_file, diff, color_diff, out, err
  43 from black.report import Report, Changed
  44 from black.files import find_project_root, find_pyproject_toml, parse_pyproject_toml
  45 from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore
  46 from black.files import wrap_stream_for_windows
  47 from black.parsing import InvalidInput  # noqa F401
  48 from black.parsing import lib2to3_parse, parse_ast, stringify_ast
  49
  50
  51 # lib2to3 fork
  52 from blib2to3.pytree import Node, Leaf
  53 from blib2to3.pgen2 import token
  54
  55 from _black_version import version as __version__
  56
  57 # If our environment has uvloop installed lets use it
  58 try:
  59     import uvloop
  60
  61     uvloop.install()
  62 except ImportError:
  63     pass
  64
  65 # types
  66 FileContent = str
  67 Encoding = str
  68 NewLine = str
  69
  70
  71 class NothingChanged(UserWarning):
  72     """Raised when reformatted code is the same as source."""
  73
  74
  75 class WriteBack(Enum):
  76     NO = 0
  77     YES = 1
  78     DIFF = 2
  79     CHECK = 3
  80     COLOR_DIFF = 4
  81
  82     @classmethod
  83     def from_configuration(
  84         cls, *, check: bool, diff: bool, color: bool = False
  85     ) -> "WriteBack":
  86         if check and not diff:
  87             return cls.CHECK
  88
  89         if diff and color:
  90             return cls.COLOR_DIFF
  91
  92         return cls.DIFF if diff else cls.YES
  93
  94
  95 # Legacy name, left for integrations.
  96 FileMode = Mode
  97
  98
  99 def read_pyproject_toml(
 100     ctx: click.Context, param: click.Parameter, value: Optional[str]
 101 ) -> Optional[str]:
 102     """Inject Black configuration from "pyproject.toml" into defaults in `ctx`.
 103
 104     Returns the path to a successfully found and read configuration file, None
 105     otherwise.
 106     """
 107     if not value:
 108         value = find_pyproject_toml(ctx.params.get("src", ()))
 109         if value is None:
 110             return None
 111
 112     try:
 113         config = parse_pyproject_toml(value)
 114     except (OSError, ValueError) as e:
 115         raise click.FileError(
 116             filename=value, hint=f"Error reading configuration file: {e}"
 117         )
 118
 119     if not config:
 120         return None
 121     else:
 122         # Sanitize the values to be Click friendly. For more information please see:
 123         # https://github.com/psf/black/issues/1458
 124         # https://github.com/pallets/click/issues/1567
 125         config = {
 126             k: str(v) if not isinstance(v, (list, dict)) else v
 127             for k, v in config.items()
 128         }
 129
 130     target_version = config.get("target_version")
 131     if target_version is not None and not isinstance(target_version, list):
 132         raise click.BadOptionUsage(
 133             "target-version", "Config key target-version must be a list"
 134         )
 135
 136     default_map: Dict[str, Any] = {}
 137     if ctx.default_map:
 138         default_map.update(ctx.default_map)
 139     default_map.update(config)
 140
 141     ctx.default_map = default_map
 142     return value
 143
 144
 145 def target_version_option_callback(
 146     c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
 147 ) -> List[TargetVersion]:
 148     """Compute the target versions from a --target-version flag.
 149
 150     This is its own function because mypy couldn't infer the type correctly
 151     when it was a lambda, causing mypyc trouble.
 152     """
 153     return [TargetVersion[val.upper()] for val in v]
 154
 155
 156 def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
 157     """Compile a regular expression string in `regex`.
 158
 159     If it contains newlines, use verbose mode.
 160     """
 161     if "\n" in regex:
 162         regex = "(?x)" + regex
 163     compiled: Pattern[str] = re.compile(regex)
 164     return compiled
 165
 166
 167 def validate_regex(
 168     ctx: click.Context,
 169     param: click.Parameter,
 170     value: Optional[str],
 171 ) -> Optional[Pattern]:
 172     try:
 173         return re_compile_maybe_verbose(value) if value is not None else None
 174     except re.error:
 175         raise click.BadParameter("Not a valid regular expression")
 176
 177
 178 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 179 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
 180 @click.option(
 181     "-l",
 182     "--line-length",
 183     type=int,
 184     default=DEFAULT_LINE_LENGTH,
 185     help="How many characters per line to allow.",
 186     show_default=True,
 187 )
 188 @click.option(
 189     "-t",
 190     "--target-version",
 191     type=click.Choice([v.name.lower() for v in TargetVersion]),
 192     callback=target_version_option_callback,
 193     multiple=True,
 194     help=(
 195         "Python versions that should be supported by Black's output. [default: per-file"
 196         " auto-detection]"
 197     ),
 198 )
 199 @click.option(
 200     "--pyi",
 201     is_flag=True,
 202     help=(
 203         "Format all input files like typing stubs regardless of file extension (useful"
 204         " when piping source on standard input)."
 205     ),
 206 )
 207 @click.option(
 208     "-S",
 209     "--skip-string-normalization",
 210     is_flag=True,
 211     help="Don't normalize string quotes or prefixes.",
 212 )
 213 @click.option(
 214     "-C",
 215     "--skip-magic-trailing-comma",
 216     is_flag=True,
 217     help="Don't use trailing commas as a reason to split lines.",
 218 )
 219 @click.option(
 220     "--experimental-string-processing",
 221     is_flag=True,
 222     hidden=True,
 223     help=(
 224         "Experimental option that performs more normalization on string literals."
 225         " Currently disabled because it leads to some crashes."
 226     ),
 227 )
 228 @click.option(
 229     "--check",
 230     is_flag=True,
 231     help=(
 232         "Don't write the files back, just return the status. Return code 0 means"
 233         " nothing would change. Return code 1 means some files would be reformatted."
 234         " Return code 123 means there was an internal error."
 235     ),
 236 )
 237 @click.option(
 238     "--diff",
 239     is_flag=True,
 240     help="Don't write the files back, just output a diff for each file on stdout.",
 241 )
 242 @click.option(
 243     "--color/--no-color",
 244     is_flag=True,
 245     help="Show colored diff. Only applies when `--diff` is given.",
 246 )
 247 @click.option(
 248     "--fast/--safe",
 249     is_flag=True,
 250     help="If --fast given, skip temporary sanity checks. [default: --safe]",
 251 )
 252 @click.option(
 253     "--include",
 254     type=str,
 255     default=DEFAULT_INCLUDES,
 256     callback=validate_regex,
 257     help=(
 258         "A regular expression that matches files and directories that should be"
 259         " included on recursive searches. An empty value means all files are included"
 260         " regardless of the name. Use forward slashes for directories on all platforms"
 261         " (Windows, too). Exclusions are calculated first, inclusions later."
 262     ),
 263     show_default=True,
 264 )
 265 @click.option(
 266     "--exclude",
 267     type=str,
 268     callback=validate_regex,
 269     help=(
 270         "A regular expression that matches files and directories that should be"
 271         " excluded on recursive searches. An empty value means no paths are excluded."
 272         " Use forward slashes for directories on all platforms (Windows, too)."
 273         " Exclusions are calculated first, inclusions later. [default:"
 274         f" {DEFAULT_EXCLUDES}]"
 275     ),
 276     show_default=False,
 277 )
 278 @click.option(
 279     "--extend-exclude",
 280     type=str,
 281     callback=validate_regex,
 282     help=(
 283         "Like --exclude, but adds additional files and directories on top of the"
 284         " excluded ones. (Useful if you simply want to add to the default)"
 285     ),
 286 )
 287 @click.option(
 288     "--force-exclude",
 289     type=str,
 290     callback=validate_regex,
 291     help=(
 292         "Like --exclude, but files and directories matching this regex will be "
 293         "excluded even when they are passed explicitly as arguments."
 294     ),
 295 )
 296 @click.option(
 297     "--stdin-filename",
 298     type=str,
 299     help=(
 300         "The name of the file when passing it through stdin. Useful to make "
 301         "sure Black will respect --force-exclude option on some "
 302         "editors that rely on using stdin."
 303     ),
 304 )
 305 @click.option(
 306     "-q",
 307     "--quiet",
 308     is_flag=True,
 309     help=(
 310         "Don't emit non-error messages to stderr. Errors are still emitted; silence"
 311         " those with 2>/dev/null."
 312     ),
 313 )
 314 @click.option(
 315     "-v",
 316     "--verbose",
 317     is_flag=True,
 318     help=(
 319         "Also emit messages to stderr about files that were not changed or were ignored"
 320         " due to exclusion patterns."
 321     ),
 322 )
 323 @click.version_option(version=__version__)
 324 @click.argument(
 325     "src",
 326     nargs=-1,
 327     type=click.Path(
 328         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
 329     ),
 330     is_eager=True,
 331 )
 332 @click.option(
 333     "--config",
 334     type=click.Path(
 335         exists=True,
 336         file_okay=True,
 337         dir_okay=False,
 338         readable=True,
 339         allow_dash=False,
 340         path_type=str,
 341     ),
 342     is_eager=True,
 343     callback=read_pyproject_toml,
 344     help="Read configuration from FILE path.",
 345 )
 346 @click.pass_context
 347 def main(
 348     ctx: click.Context,
 349     code: Optional[str],
 350     line_length: int,
 351     target_version: List[TargetVersion],
 352     check: bool,
 353     diff: bool,
 354     color: bool,
 355     fast: bool,
 356     pyi: bool,
 357     skip_string_normalization: bool,
 358     skip_magic_trailing_comma: bool,
 359     experimental_string_processing: bool,
 360     quiet: bool,
 361     verbose: bool,
 362     include: Pattern,
 363     exclude: Optional[Pattern],
 364     extend_exclude: Optional[Pattern],
 365     force_exclude: Optional[Pattern],
 366     stdin_filename: Optional[str],
 367     src: Tuple[str, ...],
 368     config: Optional[str],
 369 ) -> None:
 370     """The uncompromising code formatter."""
 371     write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)
 372     if target_version:
 373         versions = set(target_version)
 374     else:
 375         # We'll autodetect later.
 376         versions = set()
 377     mode = Mode(
 378         target_versions=versions,
 379         line_length=line_length,
 380         is_pyi=pyi,
 381         string_normalization=not skip_string_normalization,
 382         magic_trailing_comma=not skip_magic_trailing_comma,
 383         experimental_string_processing=experimental_string_processing,
 384     )
 385     if config and verbose:
 386         out(f"Using configuration from {config}.", bold=False, fg="blue")
 387
 388     if code is not None:
 389         # Run in quiet mode by default with -c; the extra output isn't useful.
 390         # You can still pass -v to get verbose output.
 391         quiet = True
 392
 393     report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
 394
 395     if code is not None:
 396         reformat_code(
 397             content=code, fast=fast, write_back=write_back, mode=mode, report=report
 398         )
 399     else:
 400         sources = get_sources(
 401             ctx=ctx,
 402             src=src,
 403             quiet=quiet,
 404             verbose=verbose,
 405             include=include,
 406             exclude=exclude,
 407             extend_exclude=extend_exclude,
 408             force_exclude=force_exclude,
 409             report=report,
 410             stdin_filename=stdin_filename,
 411         )
 412
 413         path_empty(
 414             sources,
 415             "No Python files are present to be formatted. Nothing to do 😴",
 416             quiet,
 417             verbose,
 418             ctx,
 419         )
 420
 421         if len(sources) == 1:
 422             reformat_one(
 423                 src=sources.pop(),
 424                 fast=fast,
 425                 write_back=write_back,
 426                 mode=mode,
 427                 report=report,
 428             )
 429         else:
 430             reformat_many(
 431                 sources=sources,
 432                 fast=fast,
 433                 write_back=write_back,
 434                 mode=mode,
 435                 report=report,
 436             )
 437
 438     if verbose or not quiet:
 439         out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
 440         if code is None:
 441             click.secho(str(report), err=True)
 442     ctx.exit(report.return_code)
 443
 444
 445 def get_sources(
 446     *,
 447     ctx: click.Context,
 448     src: Tuple[str, ...],
 449     quiet: bool,
 450     verbose: bool,
 451     include: Pattern[str],
 452     exclude: Optional[Pattern[str]],
 453     extend_exclude: Optional[Pattern[str]],
 454     force_exclude: Optional[Pattern[str]],
 455     report: "Report",
 456     stdin_filename: Optional[str],
 457 ) -> Set[Path]:
 458     """Compute the set of files to be formatted."""
 459
 460     root = find_project_root(src)
 461     sources: Set[Path] = set()
 462     path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx)
 463
 464     if exclude is None:
 465         exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
 466         gitignore = get_gitignore(root)
 467     else:
 468         gitignore = None
 469
 470     for s in src:
 471         if s == "-" and stdin_filename:
 472             p = Path(stdin_filename)
 473             is_stdin = True
 474         else:
 475             p = Path(s)
 476             is_stdin = False
 477
 478         if is_stdin or p.is_file():
 479             normalized_path = normalize_path_maybe_ignore(p, root, report)
 480             if normalized_path is None:
 481                 continue
 482
 483             normalized_path = "/" + normalized_path
 484             # Hard-exclude any files that matches the `--force-exclude` regex.
 485             if force_exclude:
 486                 force_exclude_match = force_exclude.search(normalized_path)
 487             else:
 488                 force_exclude_match = None
 489             if force_exclude_match and force_exclude_match.group(0):
 490                 report.path_ignored(p, "matches the --force-exclude regular expression")
 491                 continue
 492
 493             if is_stdin:
 494                 p = Path(f"{STDIN_PLACEHOLDER}{str(p)}")
 495
 496             sources.add(p)
 497         elif p.is_dir():
 498             sources.update(
 499                 gen_python_files(
 500                     p.iterdir(),
 501                     root,
 502                     include,
 503                     exclude,
 504                     extend_exclude,
 505                     force_exclude,
 506                     report,
 507                     gitignore,
 508                 )
 509             )
 510         elif s == "-":
 511             sources.add(p)
 512         else:
 513             err(f"invalid path: {s}")
 514     return sources
 515
 516
 517 def path_empty(
 518     src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context
 519 ) -> None:
 520     """
 521     Exit if there is no `src` provided for formatting
 522     """
 523     if not src:
 524         if verbose or not quiet:
 525             out(msg)
 526         ctx.exit(0)
 527
 528
 529 def reformat_code(
 530     content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report
 531 ) -> None:
 532     """
 533     Reformat and print out `content` without spawning child processes.
 534     Similar to `reformat_one`, but for string content.
 535
 536     `fast`, `write_back`, and `mode` options are passed to
 537     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
 538     """
 539     path = Path("<string>")
 540     try:
 541         changed = Changed.NO
 542         if format_stdin_to_stdout(
 543             content=content, fast=fast, write_back=write_back, mode=mode
 544         ):
 545             changed = Changed.YES
 546         report.done(path, changed)
 547     except Exception as exc:
 548         if report.verbose:
 549             traceback.print_exc()
 550         report.failed(path, str(exc))
 551
 552
 553 def reformat_one(
 554     src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 555 ) -> None:
 556     """Reformat a single file under `src` without spawning child processes.
 557
 558     `fast`, `write_back`, and `mode` options are passed to
 559     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
 560     """
 561     try:
 562         changed = Changed.NO
 563
 564         if str(src) == "-":
 565             is_stdin = True
 566         elif str(src).startswith(STDIN_PLACEHOLDER):
 567             is_stdin = True
 568             # Use the original name again in case we want to print something
 569             # to the user
 570             src = Path(str(src)[len(STDIN_PLACEHOLDER) :])
 571         else:
 572             is_stdin = False
 573
 574         if is_stdin:
 575             if src.suffix == ".pyi":
 576                 mode = replace(mode, is_pyi=True)
 577             if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode):
 578                 changed = Changed.YES
 579         else:
 580             cache: Cache = {}
 581             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 582                 cache = read_cache(mode)
 583                 res_src = src.resolve()
 584                 res_src_s = str(res_src)
 585                 if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
 586                     changed = Changed.CACHED
 587             if changed is not Changed.CACHED and format_file_in_place(
 588                 src, fast=fast, write_back=write_back, mode=mode
 589             ):
 590                 changed = Changed.YES
 591             if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
 592                 write_back is WriteBack.CHECK and changed is Changed.NO
 593             ):
 594                 write_cache(cache, [src], mode)
 595         report.done(src, changed)
 596     except Exception as exc:
 597         if report.verbose:
 598             traceback.print_exc()
 599         report.failed(src, str(exc))
 600
 601
 602 def reformat_many(
 603     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 604 ) -> None:
 605     """Reformat multiple files using a ProcessPoolExecutor."""
 606     executor: Executor
 607     loop = asyncio.get_event_loop()
 608     worker_count = os.cpu_count()
 609     if sys.platform == "win32":
 610         # Work around https://bugs.python.org/issue26903
 611         worker_count = min(worker_count, 60)
 612     try:
 613         executor = ProcessPoolExecutor(max_workers=worker_count)
 614     except (ImportError, OSError):
 615         # we arrive here if the underlying system does not support multi-processing
 616         # like in AWS Lambda or Termux, in which case we gracefully fallback to
 617         # a ThreadPoolExecutor with just a single worker (more workers would not do us
 618         # any good due to the Global Interpreter Lock)
 619         executor = ThreadPoolExecutor(max_workers=1)
 620
 621     try:
 622         loop.run_until_complete(
 623             schedule_formatting(
 624                 sources=sources,
 625                 fast=fast,
 626                 write_back=write_back,
 627                 mode=mode,
 628                 report=report,
 629                 loop=loop,
 630                 executor=executor,
 631             )
 632         )
 633     finally:
 634         shutdown(loop)
 635         if executor is not None:
 636             executor.shutdown()
 637
 638
 639 async def schedule_formatting(
 640     sources: Set[Path],
 641     fast: bool,
 642     write_back: WriteBack,
 643     mode: Mode,
 644     report: "Report",
 645     loop: asyncio.AbstractEventLoop,
 646     executor: Executor,
 647 ) -> None:
 648     """Run formatting of `sources` in parallel using the provided `executor`.
 649
 650     (Use ProcessPoolExecutors for actual parallelism.)
 651
 652     `write_back`, `fast`, and `mode` options are passed to
 653     :func:`format_file_in_place`.
 654     """
 655     cache: Cache = {}
 656     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 657         cache = read_cache(mode)
 658         sources, cached = filter_cached(cache, sources)
 659         for src in sorted(cached):
 660             report.done(src, Changed.CACHED)
 661     if not sources:
 662         return
 663
 664     cancelled = []
 665     sources_to_cache = []
 666     lock = None
 667     if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 668         # For diff output, we need locks to ensure we don't interleave output
 669         # from different processes.
 670         manager = Manager()
 671         lock = manager.Lock()
 672     tasks = {
 673         asyncio.ensure_future(
 674             loop.run_in_executor(
 675                 executor, format_file_in_place, src, fast, mode, write_back, lock
 676             )
 677         ): src
 678         for src in sorted(sources)
 679     }
 680     pending = tasks.keys()
 681     try:
 682         loop.add_signal_handler(signal.SIGINT, cancel, pending)
 683         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
 684     except NotImplementedError:
 685         # There are no good alternatives for these on Windows.
 686         pass
 687     while pending:
 688         done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
 689         for task in done:
 690             src = tasks.pop(task)
 691             if task.cancelled():
 692                 cancelled.append(task)
 693             elif task.exception():
 694                 report.failed(src, str(task.exception()))
 695             else:
 696                 changed = Changed.YES if task.result() else Changed.NO
 697                 # If the file was written back or was successfully checked as
 698                 # well-formatted, store this information in the cache.
 699                 if write_back is WriteBack.YES or (
 700                     write_back is WriteBack.CHECK and changed is Changed.NO
 701                 ):
 702                     sources_to_cache.append(src)
 703                 report.done(src, changed)
 704     if cancelled:
 705         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
 706     if sources_to_cache:
 707         write_cache(cache, sources_to_cache, mode)
 708
 709
 710 def format_file_in_place(
 711     src: Path,
 712     fast: bool,
 713     mode: Mode,
 714     write_back: WriteBack = WriteBack.NO,
 715     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
 716 ) -> bool:
 717     """Format file under `src` path. Return True if changed.
 718
 719     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
 720     code to the file.
 721     `mode` and `fast` options are passed to :func:`format_file_contents`.
 722     """
 723     if src.suffix == ".pyi":
 724         mode = replace(mode, is_pyi=True)
 725
 726     then = datetime.utcfromtimestamp(src.stat().st_mtime)
 727     with open(src, "rb") as buf:
 728         src_contents, encoding, newline = decode_bytes(buf.read())
 729     try:
 730         dst_contents = format_file_contents(src_contents, fast=fast, mode=mode)
 731     except NothingChanged:
 732         return False
 733
 734     if write_back == WriteBack.YES:
 735         with open(src, "w", encoding=encoding, newline=newline) as f:
 736             f.write(dst_contents)
 737     elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 738         now = datetime.utcnow()
 739         src_name = f"{src}\t{then} +0000"
 740         dst_name = f"{src}\t{now} +0000"
 741         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
 742
 743         if write_back == WriteBack.COLOR_DIFF:
 744             diff_contents = color_diff(diff_contents)
 745
 746         with lock or nullcontext():
 747             f = io.TextIOWrapper(
 748                 sys.stdout.buffer,
 749                 encoding=encoding,
 750                 newline=newline,
 751                 write_through=True,
 752             )
 753             f = wrap_stream_for_windows(f)
 754             f.write(diff_contents)
 755             f.detach()
 756
 757     return True
 758
 759
 760 def format_stdin_to_stdout(
 761     fast: bool,
 762     *,
 763     content: Optional[str] = None,
 764     write_back: WriteBack = WriteBack.NO,
 765     mode: Mode,
 766 ) -> bool:
 767     """Format file on stdin. Return True if changed.
 768
 769     If content is None, it's read from sys.stdin.
 770
 771     If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
 772     write a diff to stdout. The `mode` argument is passed to
 773     :func:`format_file_contents`.
 774     """
 775     then = datetime.utcnow()
 776
 777     if content is None:
 778         src, encoding, newline = decode_bytes(sys.stdin.buffer.read())
 779     else:
 780         src, encoding, newline = content, "utf-8", ""
 781
 782     dst = src
 783     try:
 784         dst = format_file_contents(src, fast=fast, mode=mode)
 785         return True
 786
 787     except NothingChanged:
 788         return False
 789
 790     finally:
 791         f = io.TextIOWrapper(
 792             sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True
 793         )
 794         if write_back == WriteBack.YES:
 795             # Make sure there's a newline after the content
 796             dst += "" if dst[-1] == "\n" else "\n"
 797             f.write(dst)
 798         elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
 799             now = datetime.utcnow()
 800             src_name = f"STDIN\t{then} +0000"
 801             dst_name = f"STDOUT\t{now} +0000"
 802             d = diff(src, dst, src_name, dst_name)
 803             if write_back == WriteBack.COLOR_DIFF:
 804                 d = color_diff(d)
 805                 f = wrap_stream_for_windows(f)
 806             f.write(d)
 807         f.detach()
 808
 809
 810 def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:
 811     """Reformat contents of a file and return new contents.
 812
 813     If `fast` is False, additionally confirm that the reformatted code is
 814     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
 815     `mode` is passed to :func:`format_str`.
 816     """
 817     if not src_contents.strip():
 818         raise NothingChanged
 819
 820     dst_contents = format_str(src_contents, mode=mode)
 821     if src_contents == dst_contents:
 822         raise NothingChanged
 823
 824     if not fast:
 825         assert_equivalent(src_contents, dst_contents)
 826
 827         # Forced second pass to work around optional trailing commas (becoming
 828         # forced trailing commas on pass 2) interacting differently with optional
 829         # parentheses.  Admittedly ugly.
 830         dst_contents_pass2 = format_str(dst_contents, mode=mode)
 831         if dst_contents != dst_contents_pass2:
 832             dst_contents = dst_contents_pass2
 833             assert_equivalent(src_contents, dst_contents, pass_num=2)
 834             assert_stable(src_contents, dst_contents, mode=mode)
 835         # Note: no need to explicitly call `assert_stable` if `dst_contents` was
 836         # the same as `dst_contents_pass2`.
 837     return dst_contents
 838
 839
 840 def format_str(src_contents: str, *, mode: Mode) -> FileContent:
 841     """Reformat a string and return new contents.
 842
 843     `mode` determines formatting options, such as how many characters per line are
 844     allowed.  Example:
 845
 846     >>> import black
 847     >>> print(black.format_str("def f(arg:str='')->None:...", mode=black.Mode()))
 848     def f(arg: str = "") -> None:
 849         ...
 850
 851     A more complex example:
 852
 853     >>> print(
 854     ...   black.format_str(
 855     ...     "def f(arg:str='')->None: hey",
 856     ...     mode=black.Mode(
 857     ...       target_versions={black.TargetVersion.PY36},
 858     ...       line_length=10,
 859     ...       string_normalization=False,
 860     ...       is_pyi=False,
 861     ...     ),
 862     ...   ),
 863     ... )
 864     def f(
 865         arg: str = '',
 866     ) -> None:
 867         hey
 868
 869     """
 870     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
 871     dst_contents = []
 872     future_imports = get_future_imports(src_node)
 873     if mode.target_versions:
 874         versions = mode.target_versions
 875     else:
 876         versions = detect_target_versions(src_node)
 877     normalize_fmt_off(src_node)
 878     lines = LineGenerator(
 879         mode=mode,
 880         remove_u_prefix="unicode_literals" in future_imports
 881         or supports_feature(versions, Feature.UNICODE_LITERALS),
 882     )
 883     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
 884     empty_line = Line(mode=mode)
 885     after = 0
 886     split_line_features = {
 887         feature
 888         for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
 889         if supports_feature(versions, feature)
 890     }
 891     for current_line in lines.visit(src_node):
 892         dst_contents.append(str(empty_line) * after)
 893         before, after = elt.maybe_empty_lines(current_line)
 894         dst_contents.append(str(empty_line) * before)
 895         for line in transform_line(
 896             current_line, mode=mode, features=split_line_features
 897         ):
 898             dst_contents.append(str(line))
 899     return "".join(dst_contents)
 900
 901
 902 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
 903     """Return a tuple of (decoded_contents, encoding, newline).
 904
 905     `newline` is either CRLF or LF but `decoded_contents` is decoded with
 906     universal newlines (i.e. only contains LF).
 907     """
 908     srcbuf = io.BytesIO(src)
 909     encoding, lines = tokenize.detect_encoding(srcbuf.readline)
 910     if not lines:
 911         return "", encoding, "\n"
 912
 913     newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n"
 914     srcbuf.seek(0)
 915     with io.TextIOWrapper(srcbuf, encoding) as tiow:
 916         return tiow.read(), encoding, newline
 917
 918
 919 def get_features_used(node: Node) -> Set[Feature]:
 920     """Return a set of (relatively) new Python features used in this file.
 921
 922     Currently looking for:
 923     - f-strings;
 924     - underscores in numeric literals;
 925     - trailing commas after * or ** in function signatures and calls;
 926     - positional only arguments in function signatures and lambdas;
 927     - assignment expression;
 928     - relaxed decorator syntax;
 929     """
 930     features: Set[Feature] = set()
 931     for n in node.pre_order():
 932         if n.type == token.STRING:
 933             value_head = n.value[:2]  # type: ignore
 934             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
 935                 features.add(Feature.F_STRINGS)
 936
 937         elif n.type == token.NUMBER:
 938             if "_" in n.value:  # type: ignore
 939                 features.add(Feature.NUMERIC_UNDERSCORES)
 940
 941         elif n.type == token.SLASH:
 942             if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
 943                 features.add(Feature.POS_ONLY_ARGUMENTS)
 944
 945         elif n.type == token.COLONEQUAL:
 946             features.add(Feature.ASSIGNMENT_EXPRESSIONS)
 947
 948         elif n.type == syms.decorator:
 949             if len(n.children) > 1 and not is_simple_decorator_expression(
 950                 n.children[1]
 951             ):
 952                 features.add(Feature.RELAXED_DECORATORS)
 953
 954         elif (
 955             n.type in {syms.typedargslist, syms.arglist}
 956             and n.children
 957             and n.children[-1].type == token.COMMA
 958         ):
 959             if n.type == syms.typedargslist:
 960                 feature = Feature.TRAILING_COMMA_IN_DEF
 961             else:
 962                 feature = Feature.TRAILING_COMMA_IN_CALL
 963
 964             for ch in n.children:
 965                 if ch.type in STARS:
 966                     features.add(feature)
 967
 968                 if ch.type == syms.argument:
 969                     for argch in ch.children:
 970                         if argch.type in STARS:
 971                             features.add(feature)
 972
 973     return features
 974
 975
 976 def detect_target_versions(node: Node) -> Set[TargetVersion]:
 977     """Detect the version to target based on the nodes used."""
 978     features = get_features_used(node)
 979     return {
 980         version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
 981     }
 982
 983
 984 def get_future_imports(node: Node) -> Set[str]:
 985     """Return a set of __future__ imports in the file."""
 986     imports: Set[str] = set()
 987
 988     def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
 989         for child in children:
 990             if isinstance(child, Leaf):
 991                 if child.type == token.NAME:
 992                     yield child.value
 993
 994             elif child.type == syms.import_as_name:
 995                 orig_name = child.children[0]
 996                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
 997                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
 998                 yield orig_name.value
 999
1000             elif child.type == syms.import_as_names:
1001                 yield from get_imports_from_children(child.children)
1002
1003             else:
1004                 raise AssertionError("Invalid syntax parsing imports")
1005
1006     for child in node.children:
1007         if child.type != syms.simple_stmt:
1008             break
1009
1010         first_child = child.children[0]
1011         if isinstance(first_child, Leaf):
1012             # Continue looking if we see a docstring; otherwise stop.
1013             if (
1014                 len(child.children) == 2
1015                 and first_child.type == token.STRING
1016                 and child.children[1].type == token.NEWLINE
1017             ):
1018                 continue
1019
1020             break
1021
1022         elif first_child.type == syms.import_from:
1023             module_name = first_child.children[1]
1024             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
1025                 break
1026
1027             imports |= set(get_imports_from_children(first_child.children[3:]))
1028         else:
1029             break
1030
1031     return imports
1032
1033
1034 def assert_equivalent(src: str, dst: str, *, pass_num: int = 1) -> None:
1035     """Raise AssertionError if `src` and `dst` aren't equivalent."""
1036     try:
1037         src_ast = parse_ast(src)
1038     except Exception as exc:
1039         raise AssertionError(
1040             "cannot use --safe with this file; failed to parse source file.  AST"
1041             f" error message: {exc}"
1042         )
1043
1044     try:
1045         dst_ast = parse_ast(dst)
1046     except Exception as exc:
1047         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
1048         raise AssertionError(
1049             f"INTERNAL ERROR: Black produced invalid code on pass {pass_num}: {exc}. "
1050             "Please report a bug on https://github.com/psf/black/issues.  "
1051             f"This invalid output might be helpful: {log}"
1052         ) from None
1053
1054     src_ast_str = "\n".join(stringify_ast(src_ast))
1055     dst_ast_str = "\n".join(stringify_ast(dst_ast))
1056     if src_ast_str != dst_ast_str:
1057         log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))
1058         raise AssertionError(
1059             "INTERNAL ERROR: Black produced code that is not equivalent to the"
1060             f" source on pass {pass_num}.  Please report a bug on "
1061             f"https://github.com/psf/black/issues.  This diff might be helpful: {log}"
1062         ) from None
1063
1064
1065 def assert_stable(src: str, dst: str, mode: Mode) -> None:
1066     """Raise AssertionError if `dst` reformats differently the second time."""
1067     newdst = format_str(dst, mode=mode)
1068     if dst != newdst:
1069         log = dump_to_file(
1070             str(mode),
1071             diff(src, dst, "source", "first pass"),
1072             diff(dst, newdst, "first pass", "second pass"),
1073         )
1074         raise AssertionError(
1075             "INTERNAL ERROR: Black produced different code on the second pass of the"
1076             " formatter.  Please report a bug on https://github.com/psf/black/issues."
1077             f"  This diff might be helpful: {log}"
1078         ) from None
1079
1080
1081 @contextmanager
1082 def nullcontext() -> Iterator[None]:
1083     """Return an empty context manager.
1084
1085     To be used like `nullcontext` in Python 3.7.
1086     """
1087     yield
1088
1089
1090 def patch_click() -> None:
1091     """Make Click not crash on Python 3.6 with LANG=C.
1092
1093     On certain misconfigured environments, Python 3 selects the ASCII encoding as the
1094     default which restricts paths that it can access during the lifetime of the
1095     application.  Click refuses to work in this scenario by raising a RuntimeError.
1096
1097     In case of Black the likelihood that non-ASCII characters are going to be used in
1098     file paths is minimal since it's Python source code.  Moreover, this crash was
1099     spurious on Python 3.7 thanks to PEP 538 and PEP 540.
1100     """
1101     try:
1102         from click import core
1103         from click import _unicodefun  # type: ignore
1104     except ModuleNotFoundError:
1105         return
1106
1107     for module in (core, _unicodefun):
1108         if hasattr(module, "_verify_python3_env"):
1109             module._verify_python3_env = lambda: None  # type: ignore
1110         if hasattr(module, "_verify_python_env"):
1111             module._verify_python_env = lambda: None  # type: ignore
1112
1113
1114 def patched_main() -> None:
1115     freeze_support()
1116     patch_click()
1117     main()
1118
1119
1120 if __name__ == "__main__":
1121     patched_main()