Test on Python 3.9-dev (#1393)

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index 087683a34f5b149ea351979d800faf07917ee43c..3ab4bc76bb23836a1f12bbe4953a5fb6bcbd7b55 100644 (file)
--- a/black.py
+++ b/black.py
@@ -2,7 +2,7 @@ import ast
 import asyncio
 from abc import ABC, abstractmethod
 from collections import defaultdict
 import asyncio
 from abc import ABC, abstractmethod
 from collections import defaultdict

-from concurrent.futures import Executor, ProcessPoolExecutor
+from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor

 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum
 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum


@@ -34,11 +34,13 @@ from typing import (
     Pattern,
     Sequence,
     Set,
     Pattern,
     Sequence,
     Set,

+    Sized,

     Tuple,
     Type,
     TypeVar,
     Union,
     cast,
     Tuple,
     Type,
     TypeVar,
     Union,
     cast,

+    TYPE_CHECKING,

 )
 from typing_extensions import Final
 from mypy_extensions import mypyc_attr
 )
 from typing_extensions import Final
 from mypy_extensions import mypyc_attr


@@ -59,6 +61,9 @@ from blib2to3.pgen2.parse import ParseError
 
 from _black_version import version as __version__
 
 
 from _black_version import version as __version__
 

+if TYPE_CHECKING:
+    import colorama  # noqa: F401
+

 DEFAULT_LINE_LENGTH = 88
 DEFAULT_EXCLUDES = r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist)/"  # noqa: B950
 DEFAULT_INCLUDES = r"\.pyi?$"
 DEFAULT_LINE_LENGTH = 88
 DEFAULT_EXCLUDES = r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist)/"  # noqa: B950
 DEFAULT_INCLUDES = r"\.pyi?$"


@@ -140,12 +145,18 @@ class WriteBack(Enum):
     YES = 1
     DIFF = 2
     CHECK = 3
     YES = 1
     DIFF = 2
     CHECK = 3

+    COLOR_DIFF = 4

 
     @classmethod
 
     @classmethod

-    def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack":
+    def from_configuration(
+        cls, *, check: bool, diff: bool, color: bool = False
+    ) -> "WriteBack":

         if check and not diff:
             return cls.CHECK
 
         if check and not diff:
             return cls.CHECK
 

+        if diff and color:
+            return cls.COLOR_DIFF
+

         return cls.DIFF if diff else cls.YES
 
 
         return cls.DIFF if diff else cls.YES
 
 


@@ -343,15 +354,6 @@ def target_version_option_callback(
         " auto-detection]"
     ),
 )
         " auto-detection]"
     ),
 )

-@click.option(
-    "--py36",
-    is_flag=True,
-    help=(
-        "Allow using Python 3.6-only syntax on all input files.  This will put trailing"
-        " commas in function signatures and calls also after *args and **kwargs."
-        " Deprecated; use --target-version instead. [default: per-file auto-detection]"
-    ),
-)

 @click.option(
     "--pyi",
     is_flag=True,
 @click.option(
     "--pyi",
     is_flag=True,


@@ -380,6 +382,11 @@ def target_version_option_callback(
     is_flag=True,
     help="Don't write the files back, just output a diff for each file on stdout.",
 )
     is_flag=True,
     help="Don't write the files back, just output a diff for each file on stdout.",
 )

+@click.option(
+    "--color/--no-color",
+    is_flag=True,
+    help="Show colored diff. Only applies when `--diff` is given.",
+)

 @click.option(
     "--fast/--safe",
     is_flag=True,
 @click.option(
     "--fast/--safe",
     is_flag=True,


@@ -409,6 +416,14 @@ def target_version_option_callback(
     ),
     show_default=True,
 )
     ),
     show_default=True,
 )

+@click.option(
+    "--force-exclude",
+    type=str,
+    help=(
+        "Like --exclude, but files and directories matching this regex will be "
+        "excluded even when they are passed explicitly as arguments"
+    ),
+)

 @click.option(
     "-q",
     "--quiet",
 @click.option(
     "-q",
     "--quiet",


@@ -458,31 +473,22 @@ def main(
     target_version: List[TargetVersion],
     check: bool,
     diff: bool,
     target_version: List[TargetVersion],
     check: bool,
     diff: bool,

+    color: bool,

     fast: bool,
     pyi: bool,
     fast: bool,
     pyi: bool,

-    py36: bool,

     skip_string_normalization: bool,
     quiet: bool,
     verbose: bool,
     include: str,
     exclude: str,
     skip_string_normalization: bool,
     quiet: bool,
     verbose: bool,
     include: str,
     exclude: str,

+    force_exclude: Optional[str],

     src: Tuple[str, ...],
     config: Optional[str],
 ) -> None:
     """The uncompromising code formatter."""
     src: Tuple[str, ...],
     config: Optional[str],
 ) -> None:
     """The uncompromising code formatter."""

-    write_back = WriteBack.from_configuration(check=check, diff=diff)
+    write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)

     if target_version:
     if target_version:

-        if py36:
-            err("Cannot use both --target-version and --py36")
-            ctx.exit(2)
-        else:
-            versions = set(target_version)
-    elif py36:
-        err(
-            "--py36 is deprecated and will be removed in a future version. Use"
-            " --target-version py36 instead."
-        )
-        versions = PY36_VERSIONS
+        versions = set(target_version)

     else:
         # We'll autodetect later.
         versions = set()
     else:
         # We'll autodetect later.
         versions = set()


@@ -497,6 +503,57 @@ def main(
     if code is not None:
         print(format_str(code, mode=mode))
         ctx.exit(0)
     if code is not None:
         print(format_str(code, mode=mode))
         ctx.exit(0)

+    report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
+    sources = get_sources(
+        ctx=ctx,
+        src=src,
+        quiet=quiet,
+        verbose=verbose,
+        include=include,
+        exclude=exclude,
+        force_exclude=force_exclude,
+        report=report,
+    )
+
+    path_empty(
+        sources,
+        "No Python files are present to be formatted. Nothing to do 😴",
+        quiet,
+        verbose,
+        ctx,
+    )
+
+    if len(sources) == 1:
+        reformat_one(
+            src=sources.pop(),
+            fast=fast,
+            write_back=write_back,
+            mode=mode,
+            report=report,
+        )
+    else:
+        reformat_many(
+            sources=sources, fast=fast, write_back=write_back, mode=mode, report=report
+        )
+
+    if verbose or not quiet:
+        out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
+        click.secho(str(report), err=True)
+    ctx.exit(report.return_code)
+
+
+def get_sources(
+    *,
+    ctx: click.Context,
+    src: Tuple[str, ...],
+    quiet: bool,
+    verbose: bool,
+    include: str,
+    exclude: str,
+    force_exclude: Optional[str],
+    report: "Report",
+) -> Set[Path]:
+    """Compute the set of files to be formatted."""

     try:
         include_regex = re_compile_maybe_verbose(include)
     except re.error:
     try:
         include_regex = re_compile_maybe_verbose(include)
     except re.error:


@@ -507,56 +564,56 @@ def main(
     except re.error:
         err(f"Invalid regular expression for exclude given: {exclude!r}")
         ctx.exit(2)
     except re.error:
         err(f"Invalid regular expression for exclude given: {exclude!r}")
         ctx.exit(2)

-    report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)
+    try:
+        force_exclude_regex = (
+            re_compile_maybe_verbose(force_exclude) if force_exclude else None
+        )
+    except re.error:
+        err(f"Invalid regular expression for force_exclude given: {force_exclude!r}")
+        ctx.exit(2)
+

     root = find_project_root(src)
     sources: Set[Path] = set()
     root = find_project_root(src)
     sources: Set[Path] = set()

-    path_empty(src, quiet, verbose, ctx)
+    path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx)
+    exclude_regexes = [exclude_regex]
+    if force_exclude_regex is not None:
+        exclude_regexes.append(force_exclude_regex)
+

     for s in src:
         p = Path(s)
         if p.is_dir():
             sources.update(
     for s in src:
         p = Path(s)
         if p.is_dir():
             sources.update(

-                gen_python_files_in_dir(
-                    p, root, include_regex, exclude_regex, report, get_gitignore(root)
+                gen_python_files(
+                    p.iterdir(),
+                    root,
+                    include_regex,
+                    exclude_regexes,
+                    report,
+                    get_gitignore(root),

                 )
             )
                 )
             )

-        elif p.is_file() or s == "-":
-            # if a file was explicitly given, we don't care about its extension
+        elif s == "-":

             sources.add(p)
             sources.add(p)

+        elif p.is_file():
+            sources.update(
+                gen_python_files(
+                    [p], root, None, exclude_regexes, report, get_gitignore(root)
+                )
+            )

         else:
             err(f"invalid path: {s}")
         else:
             err(f"invalid path: {s}")

-    if len(sources) == 0:
-        if verbose or not quiet:
-            out("No Python files are present to be formatted. Nothing to do 😴")
-        ctx.exit(0)
-
-    if len(sources) == 1:
-        reformat_one(
-            src=sources.pop(),
-            fast=fast,
-            write_back=write_back,
-            mode=mode,
-            report=report,
-        )
-    else:
-        reformat_many(
-            sources=sources, fast=fast, write_back=write_back, mode=mode, report=report
-        )
-
-    if verbose or not quiet:
-        out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
-        click.secho(str(report), err=True)
-    ctx.exit(report.return_code)
+    return sources

 
 
 def path_empty(
 
 
 def path_empty(

-    src: Tuple[str, ...], quiet: bool, verbose: bool, ctx: click.Context
+    src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context

 ) -> None:
     """
     Exit if there is no `src` provided for formatting
     """
 ) -> None:
     """
     Exit if there is no `src` provided for formatting
     """

-    if not src:
+    if len(src) == 0:

         if verbose or not quiet:
         if verbose or not quiet:

-            out("No Path provided. Nothing to do 😴")
+            out(msg)

             ctx.exit(0)
 
 
             ctx.exit(0)
 
 


@@ -597,12 +654,21 @@ def reformat_many(
     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 ) -> None:
     """Reformat multiple files using a ProcessPoolExecutor."""
     sources: Set[Path], fast: bool, write_back: WriteBack, mode: Mode, report: "Report"
 ) -> None:
     """Reformat multiple files using a ProcessPoolExecutor."""

+    executor: Executor

     loop = asyncio.get_event_loop()
     worker_count = os.cpu_count()
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)
     loop = asyncio.get_event_loop()
     worker_count = os.cpu_count()
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)

-    executor = ProcessPoolExecutor(max_workers=worker_count)
+    try:
+        executor = ProcessPoolExecutor(max_workers=worker_count)
+    except OSError:
+        # we arrive here if the underlying system does not support multi-processing
+        # like in AWS Lambda, in which case we gracefully fallback to
+        # a ThreadPollExecutor with just a single worker (more workers would not do us
+        # any good due to the Global Interpreter Lock)
+        executor = ThreadPoolExecutor(max_workers=1)
+

     try:
         loop.run_until_complete(
             schedule_formatting(
     try:
         loop.run_until_complete(
             schedule_formatting(


@@ -617,7 +683,8 @@ def reformat_many(
         )
     finally:
         shutdown(loop)
         )
     finally:
         shutdown(loop)

-        executor.shutdown()
+        if executor is not None:
+            executor.shutdown()

 
 
 async def schedule_formatting(
 
 
 async def schedule_formatting(


@@ -718,12 +785,15 @@ def format_file_in_place(
     if write_back == WriteBack.YES:
         with open(src, "w", encoding=encoding, newline=newline) as f:
             f.write(dst_contents)
     if write_back == WriteBack.YES:
         with open(src, "w", encoding=encoding, newline=newline) as f:
             f.write(dst_contents)

-    elif write_back == WriteBack.DIFF:
+    elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):

         now = datetime.utcnow()
         src_name = f"{src}\t{then} +0000"
         dst_name = f"{src}\t{now} +0000"
         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
 
         now = datetime.utcnow()
         src_name = f"{src}\t{then} +0000"
         dst_name = f"{src}\t{now} +0000"
         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
 

+        if write_back == write_back.COLOR_DIFF:
+            diff_contents = color_diff(diff_contents)
+

         with lock or nullcontext():
             f = io.TextIOWrapper(
                 sys.stdout.buffer,
         with lock or nullcontext():
             f = io.TextIOWrapper(
                 sys.stdout.buffer,


@@ -731,12 +801,57 @@ def format_file_in_place(
                 newline=newline,
                 write_through=True,
             )
                 newline=newline,
                 write_through=True,
             )

+            f = wrap_stream_for_windows(f)

             f.write(diff_contents)
             f.detach()
 
     return True
 
 
             f.write(diff_contents)
             f.detach()
 
     return True
 
 

+def color_diff(contents: str) -> str:
+    """Inject the ANSI color codes to the diff."""
+    lines = contents.split("\n")
+    for i, line in enumerate(lines):
+        if line.startswith("+++") or line.startswith("---"):
+            line = "\033[1;37m" + line + "\033[0m"  # bold white, reset
+        if line.startswith("@@"):
+            line = "\033[36m" + line + "\033[0m"  # cyan, reset
+        if line.startswith("+"):
+            line = "\033[32m" + line + "\033[0m"  # green, reset
+        elif line.startswith("-"):
+            line = "\033[31m" + line + "\033[0m"  # red, reset
+        lines[i] = line
+    return "\n".join(lines)
+
+
+def wrap_stream_for_windows(
+    f: io.TextIOWrapper,
+) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32.AnsiToWin32"]:
+    """
+    Wrap the stream in colorama's wrap_stream so colors are shown on Windows.
+
+    If `colorama` is not found, then no change is made. If `colorama` does
+    exist, then it handles the logic to determine whether or not to change
+    things.
+    """
+    try:
+        from colorama import initialise
+
+        # We set `strip=False` so that we can don't have to modify
+        # test_express_diff_with_color.
+        f = initialise.wrap_stream(
+            f, convert=None, strip=False, autoreset=False, wrap=True
+        )
+
+        # wrap_stream returns a `colorama.AnsiToWin32.AnsiToWin32` object
+        # which does not have a `detach()` method. So we fake one.
+        f.detach = lambda *args, **kwargs: None  # type: ignore
+    except ImportError:
+        pass
+
+    return f
+
+

 def format_stdin_to_stdout(
     fast: bool, *, write_back: WriteBack = WriteBack.NO, mode: Mode
 ) -> bool:
 def format_stdin_to_stdout(
     fast: bool, *, write_back: WriteBack = WriteBack.NO, mode: Mode
 ) -> bool:


@@ -762,11 +877,15 @@ def format_stdin_to_stdout(
         )
         if write_back == WriteBack.YES:
             f.write(dst)
         )
         if write_back == WriteBack.YES:
             f.write(dst)

-        elif write_back == WriteBack.DIFF:
+        elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):

             now = datetime.utcnow()
             src_name = f"STDIN\t{then} +0000"
             dst_name = f"STDOUT\t{now} +0000"
             now = datetime.utcnow()
             src_name = f"STDIN\t{then} +0000"
             dst_name = f"STDOUT\t{now} +0000"

-            f.write(diff(src, dst, src_name, dst_name))
+            d = diff(src, dst, src_name, dst_name)
+            if write_back == WriteBack.COLOR_DIFF:
+                d = color_diff(d)
+                f = wrap_stream_for_windows(f)
+            f.write(d)

         f.detach()
 
 
         f.detach()
 
 


@@ -5156,7 +5275,7 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
     """
     container: Optional[LN] = container_of(leaf)
     while container is not None and container.type != token.ENDMARKER:
     """
     container: Optional[LN] = container_of(leaf)
     while container is not None and container.type != token.ENDMARKER:

-        if fmt_on(container):
+        if is_fmt_on(container):

             return
 
         # fix for fmt: on in children
             return
 
         # fix for fmt: on in children


@@ -5170,17 +5289,21 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
             container = container.next_sibling
 
 
             container = container.next_sibling
 
 

-def fmt_on(container: LN) -> bool:
-    is_fmt_on = False
+def is_fmt_on(container: LN) -> bool:
+    """Determine whether formatting is switched on within a container.
+    Determined by whether the last `# fmt:` comment is `on` or `off`.
+    """
+    fmt_on = False

     for comment in list_comments(container.prefix, is_endmarker=False):
         if comment.value in FMT_ON:
     for comment in list_comments(container.prefix, is_endmarker=False):
         if comment.value in FMT_ON:

-            is_fmt_on = True
+            fmt_on = True

         elif comment.value in FMT_OFF:
         elif comment.value in FMT_OFF:

-            is_fmt_on = False
-    return is_fmt_on
+            fmt_on = False
+    return fmt_on

 
 
 def contains_fmt_on_at_column(container: LN, column: int) -> bool:
 
 
 def contains_fmt_on_at_column(container: LN, column: int) -> bool:

+    """Determine if children at a given column have formatting switched on."""

     for child in container.children:
         if (
             isinstance(child, Node)
     for child in container.children:
         if (
             isinstance(child, Node)


@@ -5188,13 +5311,14 @@ def contains_fmt_on_at_column(container: LN, column: int) -> bool:
             or isinstance(child, Leaf)
             and child.column == column
         ):
             or isinstance(child, Leaf)
             and child.column == column
         ):

-            if fmt_on(child):
+            if is_fmt_on(child):

                 return True
 
     return False
 
 
 def first_leaf_column(node: Node) -> Optional[int]:
                 return True
 
     return False
 
 
 def first_leaf_column(node: Node) -> Optional[int]:

+    """Returns the column of the first leaf child of a node."""

     for child in node.children:
         if isinstance(child, Leaf):
             return child.column
     for child in node.children:
         if isinstance(child, Leaf):
             return child.column


@@ -5625,11 +5749,11 @@ def get_gitignore(root: Path) -> PathSpec:
     return PathSpec.from_lines("gitwildmatch", lines)
 
 
     return PathSpec.from_lines("gitwildmatch", lines)
 
 

-def gen_python_files_in_dir(
-    path: Path,
+def gen_python_files(
+    paths: Iterable[Path],

     root: Path,
     root: Path,

-    include: Pattern[str],
-    exclude: Pattern[str],
+    include: Optional[Pattern[str]],
+    exclude_regexes: Iterable[Pattern[str]],

     report: "Report",
     gitignore: PathSpec,
 ) -> Iterator[Path]:
     report: "Report",
     gitignore: PathSpec,
 ) -> Iterator[Path]:


@@ -5641,19 +5765,13 @@ def gen_python_files_in_dir(
     `report` is where output about exclusions goes.
     """
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     `report` is where output about exclusions goes.
     """
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"

-    for child in path.iterdir():
-        # First ignore files matching .gitignore
-        if gitignore.match_file(child.as_posix()):
-            report.path_ignored(child, "matches the .gitignore file content")
-            continue
-
+    for child in paths:

         # Then ignore with `exclude` option.
         try:
         # Then ignore with `exclude` option.
         try:

-            normalized_path = "/" + child.resolve().relative_to(root).as_posix()
+            normalized_path = child.resolve().relative_to(root).as_posix()

         except OSError as e:
             report.path_ignored(child, f"cannot be read because {e}")
             continue
         except OSError as e:
             report.path_ignored(child, f"cannot be read because {e}")
             continue

-

         except ValueError:
             if child.is_symlink():
                 report.path_ignored(
         except ValueError:
             if child.is_symlink():
                 report.path_ignored(


@@ -5663,21 +5781,32 @@ def gen_python_files_in_dir(
 
             raise
 
 
             raise
 

+        # First ignore files matching .gitignore
+        if gitignore.match_file(normalized_path):
+            report.path_ignored(child, "matches the .gitignore file content")
+            continue
+
+        normalized_path = "/" + normalized_path

         if child.is_dir():
             normalized_path += "/"
 
         if child.is_dir():
             normalized_path += "/"
 

-        exclude_match = exclude.search(normalized_path)
-        if exclude_match and exclude_match.group(0):
-            report.path_ignored(child, "matches the --exclude regular expression")
+        is_excluded = False
+        for exclude in exclude_regexes:
+            exclude_match = exclude.search(normalized_path) if exclude else None
+            if exclude_match and exclude_match.group(0):
+                report.path_ignored(child, "matches the --exclude regular expression")
+                is_excluded = True
+                break
+        if is_excluded:

             continue
 
         if child.is_dir():
             continue
 
         if child.is_dir():

-            yield from gen_python_files_in_dir(
-                child, root, include, exclude, report, gitignore
+            yield from gen_python_files(
+                child.iterdir(), root, include, exclude_regexes, report, gitignore

             )
 
         elif child.is_file():
             )
 
         elif child.is_file():

-            include_match = include.search(normalized_path)
+            include_match = include.search(normalized_path) if include else True

             if include_match:
                 yield child
 
             if include_match:
                 yield child