]> git.madduck.net Git - etc/vim.git/blobdiff - tests/test_black.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

stub style: remove some possible future changes (#2940)
[etc/vim.git] / tests / test_black.py
index e17f43b88d439b7988003dea63b9cd07112d0046..b1bf1772550588d32fc6d5a8c2a630628542f1ad 100644 (file)
 #!/usr/bin/env python3
-import multiprocessing
+
 import asyncio
+import inspect
+import io
 import logging
+import multiprocessing
+import os
+import sys
+import types
+import unittest
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager
+from contextlib import contextmanager, redirect_stderr
 from dataclasses import replace
-from functools import partial
-import inspect
-from io import BytesIO, TextIOWrapper
-import os
+from io import BytesIO
 from pathlib import Path
 from platform import system
-import regex as re
-import sys
 from tempfile import TemporaryDirectory
-import types
 from typing import (
     Any,
-    BinaryIO,
     Callable,
     Dict,
-    Generator,
-    List,
-    Tuple,
     Iterator,
+    List,
+    Optional,
+    Sequence,
     TypeVar,
+    Union,
 )
-import unittest
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 import click
+import pytest
+import re
 from click import unstyle
 from click.testing import CliRunner
+from pathspec import PathSpec
 
 import black
+import black.files
 from black import Feature, TargetVersion
-
-try:
-    import blackd
-    from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop
-    from aiohttp import web
-except ImportError:
-    has_blackd_deps = False
-else:
-    has_blackd_deps = True
-
-from pathspec import PathSpec
+from black import re_compile_maybe_verbose as compile_pattern
+from black.cache import get_cache_dir, get_cache_file
+from black.debug import DebugVisitor
+from black.output import color_diff, diff
+from black.report import Report
 
 # Import other test classes
-from .test_primer import PrimerCLITests  # noqa: F401
-
+from tests.util import (
+    DATA_DIR,
+    DEFAULT_MODE,
+    DETERMINISTIC_HEADER,
+    PROJECT_ROOT,
+    PY36_VERSIONS,
+    THIS_DIR,
+    BlackBaseTestCase,
+    assert_format,
+    change_directory,
+    dump_to_stderr,
+    ff,
+    fs,
+    read_data,
+)
 
-DEFAULT_MODE = black.FileMode(experimental_string_processing=True)
-ff = partial(black.format_file_in_place, mode=DEFAULT_MODE, fast=True)
-fs = partial(black.format_str, mode=DEFAULT_MODE)
 THIS_FILE = Path(__file__)
-THIS_DIR = THIS_FILE.parent
-PROJECT_ROOT = THIS_DIR.parent
-DETERMINISTIC_HEADER = "[Deterministic header]"
-EMPTY_LINE = "# EMPTY LINE WITH WHITESPACE" + " (this comment will be removed)"
-PY36_VERSIONS = {
-    TargetVersion.PY36,
-    TargetVersion.PY37,
-    TargetVersion.PY38,
-    TargetVersion.PY39,
-}
+EMPTY_CONFIG = THIS_DIR / "data" / "empty_pyproject.toml"
 PY36_ARGS = [f"--target-version={version.name.lower()}" for version in PY36_VERSIONS]
+DEFAULT_EXCLUDE = black.re_compile_maybe_verbose(black.const.DEFAULT_EXCLUDES)
+DEFAULT_INCLUDE = black.re_compile_maybe_verbose(black.const.DEFAULT_INCLUDES)
 T = TypeVar("T")
 R = TypeVar("R")
 
-
-def dump_to_stderr(*output: str) -> str:
-    return "\n" + "\n".join(output) + "\n"
-
-
-def read_data(name: str, data: bool = True) -> Tuple[str, str]:
-    """read_data('test_name') -> 'input', 'output'"""
-    if not name.endswith((".py", ".pyi", ".out", ".diff")):
-        name += ".py"
-    _input: List[str] = []
-    _output: List[str] = []
-    base_dir = THIS_DIR / "data" if data else PROJECT_ROOT
-    with open(base_dir / name, "r", encoding="utf8") as test:
-        lines = test.readlines()
-    result = _input
-    for line in lines:
-        line = line.replace(EMPTY_LINE, "")
-        if line.rstrip() == "# output":
-            result = _output
-            continue
-
-        result.append(line)
-    if _input and not _output:
-        # If there's no output marker, treat the entire file as already pre-formatted.
-        _output = _input[:]
-    return "".join(_input).strip() + "\n", "".join(_output).strip() + "\n"
+# Match the time output in a diff, but nothing else
+DIFF_TIME = re.compile(r"\t[\d\-:+\. ]+")
 
 
 @contextmanager
@@ -103,7 +80,7 @@ def cache_dir(exists: bool = True) -> Iterator[Path]:
         cache_dir = Path(workspace)
         if not exists:
             cache_dir = cache_dir / "new"
-        with patch("black.CACHE_DIR", cache_dir):
+        with patch("black.cache.CACHE_DIR", cache_dir):
             yield cache_dir
 
 
@@ -119,22 +96,13 @@ def event_loop() -> Iterator[None]:
         loop.close()
 
 
-@contextmanager
-def skip_if_exception(e: str) -> Iterator[None]:
-    try:
-        yield
-    except Exception as exc:
-        if exc.__class__.__name__ == e:
-            unittest.skip(f"Encountered expected exception {exc}, skipping")
-        else:
-            raise
-
-
 class FakeContext(click.Context):
     """A fake click Context for when calling functions that need it."""
 
     def __init__(self) -> None:
         self.default_map: Dict[str, Any] = {}
+        # Dummy root, since most of the tests don't care about it
+        self.obj: Dict[str, Any] = {"root": PROJECT_ROOT}
 
 
 class FakeParameter(click.Parameter):
@@ -145,88 +113,32 @@ class FakeParameter(click.Parameter):
 
 
 class BlackRunner(CliRunner):
-    """Modify CliRunner so that stderr is not merged with stdout.
-
-    This is a hack that can be removed once we depend on Click 7.x"""
+    """Make sure STDOUT and STDERR are kept separate when testing Black via its CLI."""
 
     def __init__(self) -> None:
-        self.stderrbuf = BytesIO()
-        self.stdoutbuf = BytesIO()
-        self.stdout_bytes = b""
-        self.stderr_bytes = b""
-        super().__init__()
-
-    @contextmanager
-    def isolation(self, *args: Any, **kwargs: Any) -> Generator[BinaryIO, None, None]:
-        with super().isolation(*args, **kwargs) as output:
-            try:
-                hold_stderr = sys.stderr
-                sys.stderr = TextIOWrapper(self.stderrbuf, encoding=self.charset)
-                yield output
-            finally:
-                self.stdout_bytes = sys.stdout.buffer.getvalue()  # type: ignore
-                self.stderr_bytes = sys.stderr.buffer.getvalue()  # type: ignore
-                sys.stderr = hold_stderr
-
-
-class BlackTestCase(unittest.TestCase):
-    maxDiff = None
-    _diffThreshold = 2 ** 20
-
-    def assertFormatEqual(self, expected: str, actual: str) -> None:
-        if actual != expected and not os.environ.get("SKIP_AST_PRINT"):
-            bdv: black.DebugVisitor[Any]
-            black.out("Expected tree:", fg="green")
-            try:
-                exp_node = black.lib2to3_parse(expected)
-                bdv = black.DebugVisitor()
-                list(bdv.visit(exp_node))
-            except Exception as ve:
-                black.err(str(ve))
-            black.out("Actual tree:", fg="red")
-            try:
-                exp_node = black.lib2to3_parse(actual)
-                bdv = black.DebugVisitor()
-                list(bdv.visit(exp_node))
-            except Exception as ve:
-                black.err(str(ve))
-        self.assertMultiLineEqual(expected, actual)
-
-    def invokeBlack(
-        self, args: List[str], exit_code: int = 0, ignore_config: bool = True
-    ) -> None:
-        runner = BlackRunner()
-        if ignore_config:
-            args = ["--verbose", "--config", str(THIS_DIR / "empty.toml"), *args]
-        result = runner.invoke(black.main, args)
-        self.assertEqual(
-            result.exit_code,
-            exit_code,
-            msg=(
-                f"Failed with args: {args}\n"
-                f"stdout: {runner.stdout_bytes.decode()!r}\n"
-                f"stderr: {runner.stderr_bytes.decode()!r}\n"
-                f"exception: {result.exception}"
-            ),
-        )
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def checkSourceFile(self, name: str, mode: black.FileMode = DEFAULT_MODE) -> None:
-        path = THIS_DIR.parent / name
-        source, expected = read_data(str(path), data=False)
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, mode)
-        self.assertFalse(ff(path))
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_empty(self) -> None:
-        source = expected = ""
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        super().__init__(mix_stderr=False)
+
+
+def invokeBlack(
+    args: List[str], exit_code: int = 0, ignore_config: bool = True
+) -> None:
+    runner = BlackRunner()
+    if ignore_config:
+        args = ["--verbose", "--config", str(THIS_DIR / "empty.toml"), *args]
+    result = runner.invoke(black.main, args, catch_exceptions=False)
+    assert result.stdout_bytes is not None
+    assert result.stderr_bytes is not None
+    msg = (
+        f"Failed with args: {args}\n"
+        f"stdout: {result.stdout_bytes.decode()!r}\n"
+        f"stderr: {result.stderr_bytes.decode()!r}\n"
+        f"exception: {result.exception}"
+    )
+    assert result.exit_code == exit_code, msg
+
+
+class BlackTestCase(BlackBaseTestCase):
+    invokeBlack = staticmethod(invokeBlack)
 
     def test_empty_ff(self) -> None:
         expected = ""
@@ -239,56 +151,28 @@ class BlackTestCase(unittest.TestCase):
             os.unlink(tmp_file)
         self.assertFormatEqual(expected, actual)
 
-    def test_self(self) -> None:
-        self.checkSourceFile("tests/test_black.py")
-
-    def test_black(self) -> None:
-        self.checkSourceFile("src/black/__init__.py")
-
-    def test_pygram(self) -> None:
-        self.checkSourceFile("src/blib2to3/pygram.py")
-
-    def test_pytree(self) -> None:
-        self.checkSourceFile("src/blib2to3/pytree.py")
-
-    def test_conv(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/conv.py")
-
-    def test_driver(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/driver.py")
-
-    def test_grammar(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/grammar.py")
-
-    def test_literals(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/literals.py")
-
-    def test_parse(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/parse.py")
-
-    def test_pgen(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/pgen.py")
-
-    def test_tokenize(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/tokenize.py")
-
-    def test_token(self) -> None:
-        self.checkSourceFile("src/blib2to3/pgen2/token.py")
-
-    def test_setup(self) -> None:
-        self.checkSourceFile("setup.py")
+    def test_experimental_string_processing_warns(self) -> None:
+        self.assertWarns(
+            black.mode.Deprecated, black.Mode, experimental_string_processing=True
+        )
 
     def test_piping(self) -> None:
         source, expected = read_data("src/black/__init__", data=False)
         result = BlackRunner().invoke(
             black.main,
-            ["-", "--fast", f"--line-length={black.DEFAULT_LINE_LENGTH}"],
+            [
+                "-",
+                "--fast",
+                f"--line-length={black.DEFAULT_LINE_LENGTH}",
+                f"--config={EMPTY_CONFIG}",
+            ],
             input=BytesIO(source.encode("utf8")),
         )
         self.assertEqual(result.exit_code, 0)
         self.assertFormatEqual(expected, result.output)
-        black.assert_equivalent(source, result.output)
-        black.assert_stable(source, result.output, DEFAULT_MODE)
+        if source != result.output:
+            black.assert_equivalent(source, result.output)
+            black.assert_stable(source, result.output, DEFAULT_MODE)
 
     def test_piping_diff(self) -> None:
         diff_header = re.compile(
@@ -297,13 +181,12 @@ class BlackTestCase(unittest.TestCase):
         )
         source, _ = read_data("expression.py")
         expected, _ = read_data("expression.diff")
-        config = THIS_DIR / "data" / "empty_pyproject.toml"
         args = [
             "-",
             "--fast",
             f"--line-length={black.DEFAULT_LINE_LENGTH}",
             "--diff",
-            f"--config={config}",
+            f"--config={EMPTY_CONFIG}",
         ]
         result = BlackRunner().invoke(
             black.main, args, input=BytesIO(source.encode("utf8"))
@@ -315,42 +198,25 @@ class BlackTestCase(unittest.TestCase):
 
     def test_piping_diff_with_color(self) -> None:
         source, _ = read_data("expression.py")
-        config = THIS_DIR / "data" / "empty_pyproject.toml"
         args = [
             "-",
             "--fast",
             f"--line-length={black.DEFAULT_LINE_LENGTH}",
             "--diff",
             "--color",
-            f"--config={config}",
+            f"--config={EMPTY_CONFIG}",
         ]
         result = BlackRunner().invoke(
             black.main, args, input=BytesIO(source.encode("utf8"))
         )
         actual = result.output
         # Again, the contents are checked in a different test, so only look for colors.
-        self.assertIn("\033[1;37m", actual)
+        self.assertIn("\033[1m", actual)
         self.assertIn("\033[36m", actual)
         self.assertIn("\033[32m", actual)
         self.assertIn("\033[31m", actual)
         self.assertIn("\033[0m", actual)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_function(self) -> None:
-        source, expected = read_data("function")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_function2(self) -> None:
-        source, expected = read_data("function2")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
     @patch("black.dump_to_file", dump_to_stderr)
     def _test_wip(self) -> None:
         source, expected = read_data("wip")
@@ -366,52 +232,6 @@ class BlackTestCase(unittest.TestCase):
         black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, black.FileMode())
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_function_trailing_comma(self) -> None:
-        source, expected = read_data("function_trailing_comma")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @unittest.expectedFailure
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_trailing_comma_optional_parens_stability1(self) -> None:
-        source, _expected = read_data("trailing_comma_optional_parens1")
-        actual = fs(source)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @unittest.expectedFailure
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_trailing_comma_optional_parens_stability2(self) -> None:
-        source, _expected = read_data("trailing_comma_optional_parens2")
-        actual = fs(source)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @unittest.expectedFailure
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_trailing_comma_optional_parens_stability3(self) -> None:
-        source, _expected = read_data("trailing_comma_optional_parens3")
-        actual = fs(source)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_expression(self) -> None:
-        source, expected = read_data("expression")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_pep_572(self) -> None:
-        source, expected = read_data("pep_572")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        if sys.version_info >= (3, 8):
-            black.assert_equivalent(source, actual)
-
     def test_pep_572_version_detection(self) -> None:
         source, _ = read_data("pep_572")
         root = black.lib2to3_parse(source)
@@ -443,13 +263,14 @@ class BlackTestCase(unittest.TestCase):
             r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d"
         )
         try:
-            result = BlackRunner().invoke(black.main, ["--diff", str(tmp_file)])
+            result = BlackRunner().invoke(
+                black.main, ["--diff", str(tmp_file), f"--config={EMPTY_CONFIG}"]
+            )
             self.assertEqual(result.exit_code, 0)
         finally:
             os.unlink(tmp_file)
         actual = result.output
         actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
-        actual = actual.rstrip() + "\n"  # the diff output has a trailing space
         if expected != actual:
             dump = black.dump_to_file(actual)
             msg = (
@@ -465,36 +286,20 @@ class BlackTestCase(unittest.TestCase):
         tmp_file = Path(black.dump_to_file(source))
         try:
             result = BlackRunner().invoke(
-                black.main, ["--diff", "--color", str(tmp_file)]
+                black.main,
+                ["--diff", "--color", str(tmp_file), f"--config={EMPTY_CONFIG}"],
             )
         finally:
             os.unlink(tmp_file)
         actual = result.output
         # We check the contents of the diff in `test_expression_diff`. All
         # we need to check here is that color codes exist in the result.
-        self.assertIn("\033[1;37m", actual)
+        self.assertIn("\033[1m", actual)
         self.assertIn("\033[36m", actual)
         self.assertIn("\033[32m", actual)
         self.assertIn("\033[31m", actual)
         self.assertIn("\033[0m", actual)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_fstring(self) -> None:
-        source, expected = read_data("fstring")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_pep_570(self) -> None:
-        source, expected = read_data("pep_570")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        if sys.version_info >= (3, 8):
-            black.assert_equivalent(source, actual)
-
     def test_detect_pos_only_arguments(self) -> None:
         source, _ = read_data("pep_570")
         root = black.lib2to3_parse(source)
@@ -506,420 +311,105 @@ class BlackTestCase(unittest.TestCase):
     @patch("black.dump_to_file", dump_to_stderr)
     def test_string_quotes(self) -> None:
         source, expected = read_data("string_quotes")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        mode = replace(DEFAULT_MODE, string_normalization=False)
+        mode = black.Mode(preview=True)
+        assert_format(source, expected, mode)
+        mode = replace(mode, string_normalization=False)
         not_normalized = fs(source, mode=mode)
         self.assertFormatEqual(source.replace("\\\n", ""), not_normalized)
         black.assert_equivalent(source, not_normalized)
         black.assert_stable(source, not_normalized, mode=mode)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_docstring(self) -> None:
-        source, expected = read_data("docstring")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_docstring_no_string_normalization(self) -> None:
-        """Like test_docstring but with string normalization off."""
-        source, expected = read_data("docstring_no_string_normalization")
-        mode = replace(DEFAULT_MODE, string_normalization=False)
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, mode)
-
-    def test_long_strings(self) -> None:
-        """Tests for splitting long strings."""
-        source, expected = read_data("long_strings")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    def test_long_strings_flag_disabled(self) -> None:
-        """Tests for turning off the string processing logic."""
-        source, expected = read_data("long_strings_flag_disabled")
-        mode = replace(DEFAULT_MODE, experimental_string_processing=False)
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_stable(expected, actual, mode)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_long_strings__edge_case(self) -> None:
-        """Edge-case tests for splitting long strings."""
-        source, expected = read_data("long_strings__edge_case")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+    def test_skip_magic_trailing_comma(self) -> None:
+        source, _ = read_data("expression.py")
+        expected, _ = read_data("expression_skip_magic_trailing_comma.diff")
+        tmp_file = Path(black.dump_to_file(source))
+        diff_header = re.compile(
+            rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d "
+            r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d"
+        )
+        try:
+            result = BlackRunner().invoke(
+                black.main, ["-C", "--diff", str(tmp_file), f"--config={EMPTY_CONFIG}"]
+            )
+            self.assertEqual(result.exit_code, 0)
+        finally:
+            os.unlink(tmp_file)
+        actual = result.output
+        actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
+        actual = actual.rstrip() + "\n"  # the diff output has a trailing space
+        if expected != actual:
+            dump = black.dump_to_file(actual)
+            msg = (
+                "Expected diff isn't equal to the actual. If you made changes to"
+                " expression.py and this is an anticipated difference, overwrite"
+                f" tests/data/expression_skip_magic_trailing_comma.diff with {dump}"
+            )
+            self.assertEqual(expected, actual, msg)
 
     @patch("black.dump_to_file", dump_to_stderr)
-    def test_long_strings__regression(self) -> None:
-        """Regression tests for splitting long strings."""
-        source, expected = read_data("long_strings__regression")
+    def test_async_as_identifier(self) -> None:
+        source_path = (THIS_DIR / "data" / "async_as_identifier.py").resolve()
+        source, expected = read_data("async_as_identifier")
         actual = fs(source)
         self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
+        major, minor = sys.version_info[:2]
+        if major < 3 or (major <= 3 and minor < 7):
+            black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, DEFAULT_MODE)
+        # ensure black can parse this when the target is 3.6
+        self.invokeBlack([str(source_path), "--target-version", "py36"])
+        # but not on 3.7, because async/await is no longer an identifier
+        self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123)
 
     @patch("black.dump_to_file", dump_to_stderr)
-    def test_slices(self) -> None:
-        source, expected = read_data("slices")
+    def test_python37(self) -> None:
+        source_path = (THIS_DIR / "data" / "python37.py").resolve()
+        source, expected = read_data("python37")
         actual = fs(source)
         self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
+        major, minor = sys.version_info[:2]
+        if major > 3 or (major == 3 and minor >= 7):
+            black.assert_equivalent(source, actual)
         black.assert_stable(source, actual, DEFAULT_MODE)
+        # ensure black can parse this when the target is 3.7
+        self.invokeBlack([str(source_path), "--target-version", "py37"])
+        # but not on 3.6, because we use async as a reserved keyword
+        self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_percent_precedence(self) -> None:
-        source, expected = read_data("percent_precedence")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+    def test_tab_comment_indentation(self) -> None:
+        contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n"
+        contents_spc = "if 1:\n    if 2:\n        pass\n    # comment\n    pass\n"
+        self.assertFormatEqual(contents_spc, fs(contents_spc))
+        self.assertFormatEqual(contents_spc, fs(contents_tab))
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments(self) -> None:
-        source, expected = read_data("comments")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t\t# comment\n\tpass\n"
+        contents_spc = "if 1:\n    if 2:\n        pass\n        # comment\n    pass\n"
+        self.assertFormatEqual(contents_spc, fs(contents_spc))
+        self.assertFormatEqual(contents_spc, fs(contents_tab))
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments2(self) -> None:
-        source, expected = read_data("comments2")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        # mixed tabs and spaces (valid Python 2 code)
+        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t# comment\n        pass\n"
+        contents_spc = "if 1:\n    if 2:\n        pass\n    # comment\n    pass\n"
+        self.assertFormatEqual(contents_spc, fs(contents_spc))
+        self.assertFormatEqual(contents_spc, fs(contents_tab))
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments3(self) -> None:
-        source, expected = read_data("comments3")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t\t# comment\n        pass\n"
+        contents_spc = "if 1:\n    if 2:\n        pass\n        # comment\n    pass\n"
+        self.assertFormatEqual(contents_spc, fs(contents_spc))
+        self.assertFormatEqual(contents_spc, fs(contents_tab))
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments4(self) -> None:
-        source, expected = read_data("comments4")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+    def test_report_verbose(self) -> None:
+        report = Report(verbose=True)
+        out_lines = []
+        err_lines = []
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments5(self) -> None:
-        source, expected = read_data("comments5")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        def out(msg: str, **kwargs: Any) -> None:
+            out_lines.append(msg)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments6(self) -> None:
-        source, expected = read_data("comments6")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+        def err(msg: str, **kwargs: Any) -> None:
+            err_lines.append(msg)
 
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comments7(self) -> None:
-        source, expected = read_data("comments7")
-        mode = replace(DEFAULT_MODE, target_versions={black.TargetVersion.PY38})
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_comment_after_escaped_newline(self) -> None:
-        source, expected = read_data("comment_after_escaped_newline")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_cantfit(self) -> None:
-        source, expected = read_data("cantfit")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_import_spacing(self) -> None:
-        source, expected = read_data("import_spacing")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_composition(self) -> None:
-        source, expected = read_data("composition")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_composition_no_trailing_comma(self) -> None:
-        source, expected = read_data("composition_no_trailing_comma")
-        mode = replace(DEFAULT_MODE, target_versions={black.TargetVersion.PY38})
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_empty_lines(self) -> None:
-        source, expected = read_data("empty_lines")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_remove_parens(self) -> None:
-        source, expected = read_data("remove_parens")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_string_prefixes(self) -> None:
-        source, expected = read_data("string_prefixes")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_numeric_literals(self) -> None:
-        source, expected = read_data("numeric_literals")
-        mode = replace(DEFAULT_MODE, target_versions=PY36_VERSIONS)
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, mode)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_numeric_literals_ignoring_underscores(self) -> None:
-        source, expected = read_data("numeric_literals_skip_underscores")
-        mode = replace(DEFAULT_MODE, target_versions=PY36_VERSIONS)
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, mode)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_numeric_literals_py2(self) -> None:
-        source, expected = read_data("numeric_literals_py2")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python2(self) -> None:
-        source, expected = read_data("python2")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python2_print_function(self) -> None:
-        source, expected = read_data("python2_print_function")
-        mode = replace(DEFAULT_MODE, target_versions={TargetVersion.PY27})
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, mode)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python2_unicode_literals(self) -> None:
-        source, expected = read_data("python2_unicode_literals")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_stub(self) -> None:
-        mode = replace(DEFAULT_MODE, is_pyi=True)
-        source, expected = read_data("stub.pyi")
-        actual = fs(source, mode=mode)
-        self.assertFormatEqual(expected, actual)
-        black.assert_stable(source, actual, mode)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_async_as_identifier(self) -> None:
-        source_path = (THIS_DIR / "data" / "async_as_identifier.py").resolve()
-        source, expected = read_data("async_as_identifier")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major < 3 or (major <= 3 and minor < 7):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        # ensure black can parse this when the target is 3.6
-        self.invokeBlack([str(source_path), "--target-version", "py36"])
-        # but not on 3.7, because async/await is no longer an identifier
-        self.invokeBlack([str(source_path), "--target-version", "py37"], exit_code=123)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python37(self) -> None:
-        source_path = (THIS_DIR / "data" / "python37.py").resolve()
-        source, expected = read_data("python37")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major > 3 or (major == 3 and minor >= 7):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-        # ensure black can parse this when the target is 3.7
-        self.invokeBlack([str(source_path), "--target-version", "py37"])
-        # but not on 3.6, because we use async as a reserved keyword
-        self.invokeBlack([str(source_path), "--target-version", "py36"], exit_code=123)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python38(self) -> None:
-        source, expected = read_data("python38")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major > 3 or (major == 3 and minor >= 8):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_python39(self) -> None:
-        source, expected = read_data("python39")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        major, minor = sys.version_info[:2]
-        if major > 3 or (major == 3 and minor >= 9):
-            black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_fmtonoff(self) -> None:
-        source, expected = read_data("fmtonoff")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_fmtonoff2(self) -> None:
-        source, expected = read_data("fmtonoff2")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_fmtonoff3(self) -> None:
-        source, expected = read_data("fmtonoff3")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_fmtonoff4(self) -> None:
-        source, expected = read_data("fmtonoff4")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_remove_empty_parentheses_after_class(self) -> None:
-        source, expected = read_data("class_blank_parentheses")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_new_line_between_class_and_code(self) -> None:
-        source, expected = read_data("class_methods_new_line")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_bracket_match(self) -> None:
-        source, expected = read_data("bracketmatch")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_tuple_assign(self) -> None:
-        source, expected = read_data("tupleassign")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    @patch("black.dump_to_file", dump_to_stderr)
-    def test_beginning_backslash(self) -> None:
-        source, expected = read_data("beginning_backslash")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
-    def test_tab_comment_indentation(self) -> None:
-        contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t# comment\n\tpass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n    # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
-        contents_tab = "if 1:\n\tif 2:\n\t\tpass\n\t\t# comment\n\tpass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n        # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
-        # mixed tabs and spaces (valid Python 2 code)
-        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t# comment\n        pass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n    # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
-        contents_tab = "if 1:\n        if 2:\n\t\tpass\n\t\t# comment\n        pass\n"
-        contents_spc = "if 1:\n    if 2:\n        pass\n        # comment\n    pass\n"
-        self.assertFormatEqual(contents_spc, fs(contents_spc))
-        self.assertFormatEqual(contents_spc, fs(contents_tab))
-
-    def test_report_verbose(self) -> None:
-        report = black.Report(verbose=True)
-        out_lines = []
-        err_lines = []
-
-        def out(msg: str, **kwargs: Any) -> None:
-            out_lines.append(msg)
-
-        def err(msg: str, **kwargs: Any) -> None:
-            err_lines.append(msg)
-
-        with patch("black.out", out), patch("black.err", err):
+        with patch("black.output._out", out), patch("black.output._err", err):
             report.done(Path("f1"), black.Changed.NO)
             self.assertEqual(len(out_lines), 1)
             self.assertEqual(len(err_lines), 0)
@@ -1011,7 +501,7 @@ class BlackTestCase(unittest.TestCase):
             )
 
     def test_report_quiet(self) -> None:
-        report = black.Report(quiet=True)
+        report = Report(quiet=True)
         out_lines = []
         err_lines = []
 
@@ -1021,7 +511,7 @@ class BlackTestCase(unittest.TestCase):
         def err(msg: str, **kwargs: Any) -> None:
             err_lines.append(msg)
 
-        with patch("black.out", out), patch("black.err", err):
+        with patch("black.output._out", out), patch("black.output._err", err):
             report.done(Path("f1"), black.Changed.NO)
             self.assertEqual(len(out_lines), 0)
             self.assertEqual(len(err_lines), 0)
@@ -1115,7 +605,7 @@ class BlackTestCase(unittest.TestCase):
         def err(msg: str, **kwargs: Any) -> None:
             err_lines.append(msg)
 
-        with patch("black.out", out), patch("black.err", err):
+        with patch("black.output._out", out), patch("black.output._err", err):
             report.done(Path("f1"), black.Changed.NO)
             self.assertEqual(len(out_lines), 0)
             self.assertEqual(len(err_lines), 0)
@@ -1207,24 +697,15 @@ class BlackTestCase(unittest.TestCase):
 
         straddling = "x + y"
         black.lib2to3_parse(straddling)
-        black.lib2to3_parse(straddling, {TargetVersion.PY27})
         black.lib2to3_parse(straddling, {TargetVersion.PY36})
-        black.lib2to3_parse(straddling, {TargetVersion.PY27, TargetVersion.PY36})
 
         py2_only = "print x"
-        black.lib2to3_parse(py2_only)
-        black.lib2to3_parse(py2_only, {TargetVersion.PY27})
         with self.assertRaises(black.InvalidInput):
             black.lib2to3_parse(py2_only, {TargetVersion.PY36})
-        with self.assertRaises(black.InvalidInput):
-            black.lib2to3_parse(py2_only, {TargetVersion.PY27, TargetVersion.PY36})
 
         py3_only = "exec(x, end=y)"
         black.lib2to3_parse(py3_only)
-        with self.assertRaises(black.InvalidInput):
-            black.lib2to3_parse(py3_only, {TargetVersion.PY27})
         black.lib2to3_parse(py3_only, {TargetVersion.PY36})
-        black.lib2to3_parse(py3_only, {TargetVersion.PY27, TargetVersion.PY36})
 
     def test_get_features_used_decorator(self) -> None:
         # Test the feature detection of new decorator syntax
@@ -1289,6 +770,48 @@ class BlackTestCase(unittest.TestCase):
         self.assertEqual(black.get_features_used(node), set())
         node = black.lib2to3_parse(expected)
         self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("lambda a, /, b: ...")
+        self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS})
+        node = black.lib2to3_parse("def fn(a, /, b): ...")
+        self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS})
+        node = black.lib2to3_parse("def fn(): yield a, b")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("def fn(): return a, b")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("def fn(): yield *b, c")
+        self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW})
+        node = black.lib2to3_parse("def fn(): return a, *b, c")
+        self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW})
+        node = black.lib2to3_parse("x = a, *b, c")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("x: Any = regular")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("x: Any = (regular, regular)")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("x: Any = Complex(Type(1))[something]")
+        self.assertEqual(black.get_features_used(node), set())
+        node = black.lib2to3_parse("x: Tuple[int, ...] = a, b, c")
+        self.assertEqual(
+            black.get_features_used(node), {Feature.ANN_ASSIGN_EXTENDED_RHS}
+        )
+
+    def test_get_features_used_for_future_flags(self) -> None:
+        for src, features in [
+            ("from __future__ import annotations", {Feature.FUTURE_ANNOTATIONS}),
+            (
+                "from __future__ import (other, annotations)",
+                {Feature.FUTURE_ANNOTATIONS},
+            ),
+            ("a = 1 + 2\nfrom something import annotations", set()),
+            ("from __future__ import x, y", set()),
+        ]:
+            with self.subTest(src=src, features=features):
+                node = black.lib2to3_parse(src)
+                future_imports = black.get_future_imports(node)
+                self.assertEqual(
+                    black.get_features_used(node, future_imports=future_imports),
+                    features,
+                )
 
     def test_get_future_imports(self) -> None:
         node = black.lib2to3_parse("\n")
@@ -1320,6 +843,7 @@ class BlackTestCase(unittest.TestCase):
         )
         self.assertEqual({"unicode_literals", "print"}, black.get_future_imports(node))
 
+    @pytest.mark.incompatible_with_mypyc
     def test_debug_visitor(self) -> None:
         source, _ = read_data("debug_visitor.py")
         expected, _ = read_data("debug_visitor.out")
@@ -1332,8 +856,8 @@ class BlackTestCase(unittest.TestCase):
         def err(msg: str, **kwargs: Any) -> None:
             err_lines.append(msg)
 
-        with patch("black.out", out), patch("black.err", err):
-            black.DebugVisitor.show(source)
+        with patch("black.debug.out", out):
+            DebugVisitor.show(source)
         actual = "\n".join(out_lines) + "\n"
         log_name = ""
         if expected != actual:
@@ -1370,6 +894,7 @@ class BlackTestCase(unittest.TestCase):
         self.assertEqual(len(n.children), 1)
         self.assertEqual(n.children[0].type, black.token.ENDMARKER)
 
+    @pytest.mark.incompatible_with_mypyc
     @unittest.skipIf(os.environ.get("SKIP_AST_PRINT"), "user set SKIP_AST_PRINT")
     def test_assertFormatEqual(self) -> None:
         out_lines = []
@@ -1381,199 +906,25 @@ class BlackTestCase(unittest.TestCase):
         def err(msg: str, **kwargs: Any) -> None:
             err_lines.append(msg)
 
-        with patch("black.out", out), patch("black.err", err):
+        with patch("black.output._out", out), patch("black.output._err", err):
             with self.assertRaises(AssertionError):
                 self.assertFormatEqual("j = [1, 2, 3]", "j = [1, 2, 3,]")
 
         out_str = "".join(out_lines)
-        self.assertTrue("Expected tree:" in out_str)
-        self.assertTrue("Actual tree:" in out_str)
+        self.assertIn("Expected tree:", out_str)
+        self.assertIn("Actual tree:", out_str)
         self.assertEqual("".join(err_lines), "")
 
-    def test_cache_broken_file(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace:
-            cache_file = black.get_cache_file(mode)
-            with cache_file.open("w") as fobj:
-                fobj.write("this is not a pickle")
-            self.assertEqual(black.read_cache(mode), {})
-            src = (workspace / "test.py").resolve()
-            with src.open("w") as fobj:
-                fobj.write("print('hello')")
-            self.invokeBlack([str(src)])
-            cache = black.read_cache(mode)
-            self.assertIn(src, cache)
-
-    def test_cache_single_file_already_cached(self) -> None:
-        mode = DEFAULT_MODE
+    @event_loop()
+    @patch("black.ProcessPoolExecutor", MagicMock(side_effect=OSError))
+    def test_works_in_mono_process_only_environment(self) -> None:
         with cache_dir() as workspace:
-            src = (workspace / "test.py").resolve()
-            with src.open("w") as fobj:
-                fobj.write("print('hello')")
-            black.write_cache({}, [src], mode)
-            self.invokeBlack([str(src)])
-            with src.open("r") as fobj:
-                self.assertEqual(fobj.read(), "print('hello')")
-
-    @event_loop()
-    def test_cache_multiple_files(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace, patch(
-            "black.ProcessPoolExecutor", new=ThreadPoolExecutor
-        ):
-            one = (workspace / "one.py").resolve()
-            with one.open("w") as fobj:
-                fobj.write("print('hello')")
-            two = (workspace / "two.py").resolve()
-            with two.open("w") as fobj:
-                fobj.write("print('hello')")
-            black.write_cache({}, [one], mode)
-            self.invokeBlack([str(workspace)])
-            with one.open("r") as fobj:
-                self.assertEqual(fobj.read(), "print('hello')")
-            with two.open("r") as fobj:
-                self.assertEqual(fobj.read(), 'print("hello")\n')
-            cache = black.read_cache(mode)
-            self.assertIn(one, cache)
-            self.assertIn(two, cache)
-
-    def test_no_cache_when_writeback_diff(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace:
-            src = (workspace / "test.py").resolve()
-            with src.open("w") as fobj:
-                fobj.write("print('hello')")
-            with patch("black.read_cache") as read_cache, patch(
-                "black.write_cache"
-            ) as write_cache:
-                self.invokeBlack([str(src), "--diff"])
-                cache_file = black.get_cache_file(mode)
-                self.assertFalse(cache_file.exists())
-                write_cache.assert_not_called()
-                read_cache.assert_not_called()
-
-    def test_no_cache_when_writeback_color_diff(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace:
-            src = (workspace / "test.py").resolve()
-            with src.open("w") as fobj:
-                fobj.write("print('hello')")
-            with patch("black.read_cache") as read_cache, patch(
-                "black.write_cache"
-            ) as write_cache:
-                self.invokeBlack([str(src), "--diff", "--color"])
-                cache_file = black.get_cache_file(mode)
-                self.assertFalse(cache_file.exists())
-                write_cache.assert_not_called()
-                read_cache.assert_not_called()
-
-    @event_loop()
-    def test_output_locking_when_writeback_diff(self) -> None:
-        with cache_dir() as workspace:
-            for tag in range(0, 4):
-                src = (workspace / f"test{tag}.py").resolve()
-                with src.open("w") as fobj:
-                    fobj.write("print('hello')")
-            with patch("black.Manager", wraps=multiprocessing.Manager) as mgr:
-                self.invokeBlack(["--diff", str(workspace)], exit_code=0)
-                # this isn't quite doing what we want, but if it _isn't_
-                # called then we cannot be using the lock it provides
-                mgr.assert_called()
-
-    @event_loop()
-    def test_output_locking_when_writeback_color_diff(self) -> None:
-        with cache_dir() as workspace:
-            for tag in range(0, 4):
-                src = (workspace / f"test{tag}.py").resolve()
-                with src.open("w") as fobj:
-                    fobj.write("print('hello')")
-            with patch("black.Manager", wraps=multiprocessing.Manager) as mgr:
-                self.invokeBlack(["--diff", "--color", str(workspace)], exit_code=0)
-                # this isn't quite doing what we want, but if it _isn't_
-                # called then we cannot be using the lock it provides
-                mgr.assert_called()
-
-    def test_no_cache_when_stdin(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir():
-            result = CliRunner().invoke(
-                black.main, ["-"], input=BytesIO(b"print('hello')")
-            )
-            self.assertEqual(result.exit_code, 0)
-            cache_file = black.get_cache_file(mode)
-            self.assertFalse(cache_file.exists())
-
-    def test_read_cache_no_cachefile(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir():
-            self.assertEqual(black.read_cache(mode), {})
-
-    def test_write_cache_read_cache(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace:
-            src = (workspace / "test.py").resolve()
-            src.touch()
-            black.write_cache({}, [src], mode)
-            cache = black.read_cache(mode)
-            self.assertIn(src, cache)
-            self.assertEqual(cache[src], black.get_cache_info(src))
-
-    def test_filter_cached(self) -> None:
-        with TemporaryDirectory() as workspace:
-            path = Path(workspace)
-            uncached = (path / "uncached").resolve()
-            cached = (path / "cached").resolve()
-            cached_but_changed = (path / "changed").resolve()
-            uncached.touch()
-            cached.touch()
-            cached_but_changed.touch()
-            cache = {cached: black.get_cache_info(cached), cached_but_changed: (0.0, 0)}
-            todo, done = black.filter_cached(
-                cache, {uncached, cached, cached_but_changed}
-            )
-            self.assertEqual(todo, {uncached, cached_but_changed})
-            self.assertEqual(done, {cached})
-
-    def test_write_cache_creates_directory_if_needed(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir(exists=False) as workspace:
-            self.assertFalse(workspace.exists())
-            black.write_cache({}, [], mode)
-            self.assertTrue(workspace.exists())
-
-    @event_loop()
-    def test_failed_formatting_does_not_get_cached(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir() as workspace, patch(
-            "black.ProcessPoolExecutor", new=ThreadPoolExecutor
-        ):
-            failing = (workspace / "failing.py").resolve()
-            with failing.open("w") as fobj:
-                fobj.write("not actually python")
-            clean = (workspace / "clean.py").resolve()
-            with clean.open("w") as fobj:
-                fobj.write('print("hello")\n')
-            self.invokeBlack([str(workspace)], exit_code=123)
-            cache = black.read_cache(mode)
-            self.assertNotIn(failing, cache)
-            self.assertIn(clean, cache)
-
-    def test_write_cache_write_fail(self) -> None:
-        mode = DEFAULT_MODE
-        with cache_dir(), patch.object(Path, "open") as mock:
-            mock.side_effect = OSError
-            black.write_cache({}, [], mode)
-
-    @event_loop()
-    @patch("black.ProcessPoolExecutor", MagicMock(side_effect=OSError))
-    def test_works_in_mono_process_only_environment(self) -> None:
-        with cache_dir() as workspace:
-            for f in [
-                (workspace / "one.py").resolve(),
-                (workspace / "two.py").resolve(),
-            ]:
-                f.write_text('print("hello")\n')
-            self.invokeBlack([str(workspace)])
+            for f in [
+                (workspace / "one.py").resolve(),
+                (workspace / "two.py").resolve(),
+            ]:
+                f.write_text('print("hello")\n')
+            self.invokeBlack([str(workspace)])
 
     @event_loop()
     def test_check_diff_use_together(self) -> None:
@@ -1587,39 +938,23 @@ class BlackTestCase(unittest.TestCase):
             # Multi file command.
             self.invokeBlack([str(src1), str(src2), "--diff", "--check"], exit_code=1)
 
-    def test_no_files(self) -> None:
+    def test_no_src_fails(self) -> None:
+        with cache_dir():
+            self.invokeBlack([], exit_code=1)
+
+    def test_src_and_code_fails(self) -> None:
         with cache_dir():
-            # Without an argument, black exits with error code 0.
-            self.invokeBlack([])
+            self.invokeBlack([".", "-c", "0"], exit_code=1)
 
     def test_broken_symlink(self) -> None:
         with cache_dir() as workspace:
             symlink = workspace / "broken_link.py"
             try:
                 symlink.symlink_to("nonexistent.py")
-            except OSError as e:
+            except (OSError, NotImplementedError) as e:
                 self.skipTest(f"Can't create symlinks: {e}")
             self.invokeBlack([str(workspace.resolve())])
 
-    def test_read_cache_line_lengths(self) -> None:
-        mode = DEFAULT_MODE
-        short_mode = replace(DEFAULT_MODE, line_length=1)
-        with cache_dir() as workspace:
-            path = (workspace / "file.py").resolve()
-            path.touch()
-            black.write_cache({}, [path], mode)
-            one = black.read_cache(mode)
-            self.assertIn(path, one)
-            two = black.read_cache(short_mode)
-            self.assertNotIn(path, two)
-
-    def test_tricky_unicode_symbols(self) -> None:
-        source, expected = read_data("tricky_unicode_symbols")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
-
     def test_single_file_force_pyi(self) -> None:
         pyi_mode = replace(DEFAULT_MODE, is_pyi=True)
         contents, expected = read_data("force_pyi")
@@ -1632,9 +967,9 @@ class BlackTestCase(unittest.TestCase):
                 actual = fh.read()
             # verify cache with --pyi is separate
             pyi_cache = black.read_cache(pyi_mode)
-            self.assertIn(path, pyi_cache)
+            self.assertIn(str(path), pyi_cache)
             normal_cache = black.read_cache(DEFAULT_MODE)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(contents, actual)
         black.assert_stable(contents, actual, pyi_mode)
@@ -1661,8 +996,8 @@ class BlackTestCase(unittest.TestCase):
             pyi_cache = black.read_cache(pyi_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_pyi(self) -> None:
         source, expected = read_data("force_pyi")
@@ -1686,9 +1021,9 @@ class BlackTestCase(unittest.TestCase):
                 actual = fh.read()
             # verify cache with --target-version is separate
             py36_cache = black.read_cache(py36_mode)
-            self.assertIn(path, py36_cache)
+            self.assertIn(str(path), py36_cache)
             normal_cache = black.read_cache(reg_mode)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertEqual(actual, expected)
 
     @event_loop()
@@ -1713,15 +1048,8 @@ class BlackTestCase(unittest.TestCase):
             pyi_cache = black.read_cache(py36_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
-
-    def test_collections(self) -> None:
-        source, expected = read_data("collections")
-        actual = fs(source)
-        self.assertFormatEqual(expected, actual)
-        black.assert_equivalent(source, actual)
-        black.assert_stable(source, actual, DEFAULT_MODE)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_py36(self) -> None:
         source, expected = read_data("force_py36")
@@ -1734,133 +1062,168 @@ class BlackTestCase(unittest.TestCase):
         actual = result.output
         self.assertFormatEqual(actual, expected)
 
-    def test_include_exclude(self) -> None:
-        path = THIS_DIR / "data" / "include_exclude_tests"
-        include = re.compile(r"\.pyi?$")
-        exclude = re.compile(r"/exclude/|/\.definitely_exclude/")
-        report = black.Report()
-        gitignore = PathSpec.from_lines("gitwildmatch", [])
-        sources: List[Path] = []
-        expected = [
-            Path(path / "b/dont_exclude/a.py"),
-            Path(path / "b/dont_exclude/a.pyi"),
-        ]
-        this_abs = THIS_DIR.resolve()
-        sources.extend(
-            black.gen_python_files(
-                path.iterdir(), this_abs, include, exclude, None, report, gitignore
+    @pytest.mark.incompatible_with_mypyc
+    def test_reformat_one_with_stdin(self) -> None:
+        with patch(
+            "black.format_stdin_to_stdout",
+            return_value=lambda *args, **kwargs: black.Changed.YES,
+        ) as fsts:
+            report = MagicMock()
+            path = Path("-")
+            black.reformat_one(
+                path,
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=DEFAULT_MODE,
+                report=report,
             )
-        )
-        self.assertEqual(sorted(expected), sorted(sources))
-
-    @patch("black.find_project_root", lambda *args: THIS_DIR.resolve())
-    def test_exclude_for_issue_1572(self) -> None:
-        # Exclude shouldn't touch files that were explicitly given to Black through the
-        # CLI. Exclude is supposed to only apply to the recursive discovery of files.
-        # https://github.com/psf/black/issues/1572
-        path = THIS_DIR / "data" / "include_exclude_tests"
-        include = ""
-        exclude = r"/exclude/|a\.py"
-        src = str(path / "b/exclude/a.py")
-        report = black.Report()
-        expected = [Path(path / "b/exclude/a.py")]
-        sources = list(
-            black.get_sources(
-                ctx=FakeContext(),
-                src=(src,),
-                quiet=True,
-                verbose=False,
-                include=include,
-                exclude=exclude,
-                force_exclude=None,
+            fsts.assert_called_once()
+            report.done.assert_called_with(path, black.Changed.YES)
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_reformat_one_with_stdin_filename(self) -> None:
+        with patch(
+            "black.format_stdin_to_stdout",
+            return_value=lambda *args, **kwargs: black.Changed.YES,
+        ) as fsts:
+            report = MagicMock()
+            p = "foo.py"
+            path = Path(f"__BLACK_STDIN_FILENAME__{p}")
+            expected = Path(p)
+            black.reformat_one(
+                path,
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=DEFAULT_MODE,
                 report=report,
             )
-        )
-        self.assertEqual(sorted(expected), sorted(sources))
+            fsts.assert_called_once_with(
+                fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE
+            )
+            # __BLACK_STDIN_FILENAME__ should have been stripped
+            report.done.assert_called_with(expected, black.Changed.YES)
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_reformat_one_with_stdin_filename_pyi(self) -> None:
+        with patch(
+            "black.format_stdin_to_stdout",
+            return_value=lambda *args, **kwargs: black.Changed.YES,
+        ) as fsts:
+            report = MagicMock()
+            p = "foo.pyi"
+            path = Path(f"__BLACK_STDIN_FILENAME__{p}")
+            expected = Path(p)
+            black.reformat_one(
+                path,
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=DEFAULT_MODE,
+                report=report,
+            )
+            fsts.assert_called_once_with(
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=replace(DEFAULT_MODE, is_pyi=True),
+            )
+            # __BLACK_STDIN_FILENAME__ should have been stripped
+            report.done.assert_called_with(expected, black.Changed.YES)
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_reformat_one_with_stdin_filename_ipynb(self) -> None:
+        with patch(
+            "black.format_stdin_to_stdout",
+            return_value=lambda *args, **kwargs: black.Changed.YES,
+        ) as fsts:
+            report = MagicMock()
+            p = "foo.ipynb"
+            path = Path(f"__BLACK_STDIN_FILENAME__{p}")
+            expected = Path(p)
+            black.reformat_one(
+                path,
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=DEFAULT_MODE,
+                report=report,
+            )
+            fsts.assert_called_once_with(
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=replace(DEFAULT_MODE, is_ipynb=True),
+            )
+            # __BLACK_STDIN_FILENAME__ should have been stripped
+            report.done.assert_called_with(expected, black.Changed.YES)
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_reformat_one_with_stdin_and_existing_path(self) -> None:
+        with patch(
+            "black.format_stdin_to_stdout",
+            return_value=lambda *args, **kwargs: black.Changed.YES,
+        ) as fsts:
+            report = MagicMock()
+            # Even with an existing file, since we are forcing stdin, black
+            # should output to stdout and not modify the file inplace
+            p = Path(str(THIS_DIR / "data/collections.py"))
+            # Make sure is_file actually returns True
+            self.assertTrue(p.is_file())
+            path = Path(f"__BLACK_STDIN_FILENAME__{p}")
+            expected = Path(p)
+            black.reformat_one(
+                path,
+                fast=True,
+                write_back=black.WriteBack.YES,
+                mode=DEFAULT_MODE,
+                report=report,
+            )
+            fsts.assert_called_once()
+            # __BLACK_STDIN_FILENAME__ should have been stripped
+            report.done.assert_called_with(expected, black.Changed.YES)
 
-    def test_gitignore_exclude(self) -> None:
-        path = THIS_DIR / "data" / "include_exclude_tests"
-        include = re.compile(r"\.pyi?$")
-        exclude = re.compile(r"")
-        report = black.Report()
-        gitignore = PathSpec.from_lines(
-            "gitwildmatch", ["exclude/", ".definitely_exclude"]
+    def test_reformat_one_with_stdin_empty(self) -> None:
+        output = io.StringIO()
+        with patch("io.TextIOWrapper", lambda *args, **kwargs: output):
+            try:
+                black.format_stdin_to_stdout(
+                    fast=True,
+                    content="",
+                    write_back=black.WriteBack.YES,
+                    mode=DEFAULT_MODE,
+                )
+            except io.UnsupportedOperation:
+                pass  # StringIO does not support detach
+            assert output.getvalue() == ""
+
+    def test_invalid_cli_regex(self) -> None:
+        for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]:
+            self.invokeBlack(["-", option, "**()(!!*)"], exit_code=2)
+
+    def test_required_version_matches_version(self) -> None:
+        self.invokeBlack(
+            ["--required-version", black.__version__, "-c", "0"],
+            exit_code=0,
+            ignore_config=True,
         )
-        sources: List[Path] = []
-        expected = [
-            Path(path / "b/dont_exclude/a.py"),
-            Path(path / "b/dont_exclude/a.pyi"),
-        ]
-        this_abs = THIS_DIR.resolve()
-        sources.extend(
-            black.gen_python_files(
-                path.iterdir(), this_abs, include, exclude, None, report, gitignore
-            )
+
+    def test_required_version_matches_partial_version(self) -> None:
+        self.invokeBlack(
+            ["--required-version", black.__version__.split(".")[0], "-c", "0"],
+            exit_code=0,
+            ignore_config=True,
         )
-        self.assertEqual(sorted(expected), sorted(sources))
 
-    def test_empty_include(self) -> None:
-        path = THIS_DIR / "data" / "include_exclude_tests"
-        report = black.Report()
-        gitignore = PathSpec.from_lines("gitwildmatch", [])
-        empty = re.compile(r"")
-        sources: List[Path] = []
-        expected = [
-            Path(path / "b/exclude/a.pie"),
-            Path(path / "b/exclude/a.py"),
-            Path(path / "b/exclude/a.pyi"),
-            Path(path / "b/dont_exclude/a.pie"),
-            Path(path / "b/dont_exclude/a.py"),
-            Path(path / "b/dont_exclude/a.pyi"),
-            Path(path / "b/.definitely_exclude/a.pie"),
-            Path(path / "b/.definitely_exclude/a.py"),
-            Path(path / "b/.definitely_exclude/a.pyi"),
-        ]
-        this_abs = THIS_DIR.resolve()
-        sources.extend(
-            black.gen_python_files(
-                path.iterdir(),
-                this_abs,
-                empty,
-                re.compile(black.DEFAULT_EXCLUDES),
-                None,
-                report,
-                gitignore,
-            )
+    def test_required_version_does_not_match_on_minor_version(self) -> None:
+        self.invokeBlack(
+            ["--required-version", black.__version__.split(".")[0] + ".999", "-c", "0"],
+            exit_code=1,
+            ignore_config=True,
         )
-        self.assertEqual(sorted(expected), sorted(sources))
 
-    def test_empty_exclude(self) -> None:
-        path = THIS_DIR / "data" / "include_exclude_tests"
-        report = black.Report()
-        gitignore = PathSpec.from_lines("gitwildmatch", [])
-        empty = re.compile(r"")
-        sources: List[Path] = []
-        expected = [
-            Path(path / "b/dont_exclude/a.py"),
-            Path(path / "b/dont_exclude/a.pyi"),
-            Path(path / "b/exclude/a.py"),
-            Path(path / "b/exclude/a.pyi"),
-            Path(path / "b/.definitely_exclude/a.py"),
-            Path(path / "b/.definitely_exclude/a.pyi"),
-        ]
-        this_abs = THIS_DIR.resolve()
-        sources.extend(
-            black.gen_python_files(
-                path.iterdir(),
-                this_abs,
-                re.compile(black.DEFAULT_INCLUDES),
-                empty,
-                None,
-                report,
-                gitignore,
-            )
+    def test_required_version_does_not_match_version(self) -> None:
+        result = BlackRunner().invoke(
+            black.main,
+            ["--required-version", "20.99b", "-c", "0"],
         )
-        self.assertEqual(sorted(expected), sorted(sources))
-
-    def test_invalid_include_exclude(self) -> None:
-        for option in ["--include", "--exclude"]:
-            self.invokeBlack(["-", option, "**()(!!*)"], exit_code=2)
+        self.assertEqual(result.exit_code, 1)
+        self.assertIn("required version", result.stderr)
 
     def test_preserves_line_endings(self) -> None:
         with TemporaryDirectory() as workspace:
@@ -1882,7 +1245,7 @@ class BlackTestCase(unittest.TestCase):
                 black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf8"))
             )
             self.assertEqual(result.exit_code, 0)
-            output = runner.stdout_bytes
+            output = result.stdout_bytes
             self.assertIn(nl.encode("utf8"), output)
             if nl == "\n":
                 self.assertNotIn(b"\r\n", output)
@@ -1891,50 +1254,9 @@ class BlackTestCase(unittest.TestCase):
         with self.assertRaises(AssertionError):
             black.assert_equivalent("{}", "None")
 
-    def test_symlink_out_of_root_directory(self) -> None:
-        path = MagicMock()
-        root = THIS_DIR.resolve()
-        child = MagicMock()
-        include = re.compile(black.DEFAULT_INCLUDES)
-        exclude = re.compile(black.DEFAULT_EXCLUDES)
-        report = black.Report()
-        gitignore = PathSpec.from_lines("gitwildmatch", [])
-        # `child` should behave like a symlink which resolved path is clearly
-        # outside of the `root` directory.
-        path.iterdir.return_value = [child]
-        child.resolve.return_value = Path("/a/b/c")
-        child.as_posix.return_value = "/a/b/c"
-        child.is_symlink.return_value = True
-        try:
-            list(
-                black.gen_python_files(
-                    path.iterdir(), root, include, exclude, None, report, gitignore
-                )
-            )
-        except ValueError as ve:
-            self.fail(f"`get_python_files_in_dir()` failed: {ve}")
-        path.iterdir.assert_called_once()
-        child.resolve.assert_called_once()
-        child.is_symlink.assert_called_once()
-        # `child` should behave like a strange file which resolved path is clearly
-        # outside of the `root` directory.
-        child.is_symlink.return_value = False
-        with self.assertRaises(ValueError):
-            list(
-                black.gen_python_files(
-                    path.iterdir(), root, include, exclude, None, report, gitignore
-                )
-            )
-        path.iterdir.assert_called()
-        self.assertEqual(path.iterdir.call_count, 2)
-        child.resolve.assert_called()
-        self.assertEqual(child.resolve.call_count, 2)
-        child.is_symlink.assert_called()
-        self.assertEqual(child.is_symlink.call_count, 2)
-
     def test_shhh_click(self) -> None:
         try:
-            from click import _unicodefun  # type: ignore
+            from click import _unicodefun
         except ModuleNotFoundError:
             self.skipTest("Incompatible Click version")
         if not hasattr(_unicodefun, "_verify_python3_env"):
@@ -1943,14 +1265,14 @@ class BlackTestCase(unittest.TestCase):
         with patch("locale.getpreferredencoding") as gpe:
             gpe.return_value = "ASCII"
             with self.assertRaises(RuntimeError):
-                _unicodefun._verify_python3_env()
+                _unicodefun._verify_python3_env()  # type: ignore
         # Now, let's silence Click...
         black.patch_click()
         # ...and confirm it's silent.
         with patch("locale.getpreferredencoding") as gpe:
             gpe.return_value = "ASCII"
             try:
-                _unicodefun._verify_python3_env()
+                _unicodefun._verify_python3_env()  # type: ignore
             except RuntimeError as re:
                 self.fail(f"`patch_click()` failed, exception still raised: {re}")
 
@@ -1967,15 +1289,7 @@ class BlackTestCase(unittest.TestCase):
             critical=fail,
             log=fail,
         ):
-            ff(THIS_FILE)
-
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    def test_blackd_main(self) -> None:
-        with patch("blackd.web.run_app"):
-            result = CliRunner().invoke(blackd.main, [])
-            if result.exception is not None:
-                raise result.exception
-            self.assertEqual(result.exit_code, 0)
+            ff(THIS_DIR / "util.py")
 
     def test_invalid_config_return_code(self) -> None:
         tmp_file = Path(black.dump_to_file())
@@ -1996,6 +1310,7 @@ class BlackTestCase(unittest.TestCase):
         self.assertEqual(config["color"], True)
         self.assertEqual(config["line_length"], 79)
         self.assertEqual(config["target_version"], ["py36", "py37", "py38"])
+        self.assertEqual(config["python_cell_magics"], ["custom1", "custom2"])
         self.assertEqual(config["exclude"], r"\.pyi?$")
         self.assertEqual(config["include"], r"\.py?$")
 
@@ -2013,6 +1328,7 @@ class BlackTestCase(unittest.TestCase):
         self.assertEqual(config["exclude"], r"\.pyi?$")
         self.assertEqual(config["include"], r"\.py?$")
 
+    @pytest.mark.incompatible_with_mypyc
     def test_find_project_root(self) -> None:
         with TemporaryDirectory() as workspace:
             root = Path(workspace)
@@ -2030,202 +1346,768 @@ class BlackTestCase(unittest.TestCase):
             src_python.touch()
 
             self.assertEqual(
-                black.find_project_root((src_dir, test_dir)), root.resolve()
+                black.find_project_root((src_dir, test_dir)),
+                (root.resolve(), "pyproject.toml"),
+            )
+            self.assertEqual(
+                black.find_project_root((src_dir,)),
+                (src_dir.resolve(), "pyproject.toml"),
+            )
+            self.assertEqual(
+                black.find_project_root((src_python,)),
+                (src_dir.resolve(), "pyproject.toml"),
+            )
+
+    @patch(
+        "black.files.find_user_pyproject_toml",
+    )
+    def test_find_pyproject_toml(self, find_user_pyproject_toml: MagicMock) -> None:
+        find_user_pyproject_toml.side_effect = RuntimeError()
+
+        with redirect_stderr(io.StringIO()) as stderr:
+            result = black.files.find_pyproject_toml(
+                path_search_start=(str(Path.cwd().root),)
+            )
+
+        assert result is None
+        err = stderr.getvalue()
+        assert "Ignoring user configuration" in err
+
+    @patch(
+        "black.files.find_user_pyproject_toml",
+        black.files.find_user_pyproject_toml.__wrapped__,
+    )
+    def test_find_user_pyproject_toml_linux(self) -> None:
+        if system() == "Windows":
+            return
+
+        # Test if XDG_CONFIG_HOME is checked
+        with TemporaryDirectory() as workspace:
+            tmp_user_config = Path(workspace) / "black"
+            with patch.dict("os.environ", {"XDG_CONFIG_HOME": workspace}):
+                self.assertEqual(
+                    black.files.find_user_pyproject_toml(), tmp_user_config.resolve()
+                )
+
+        # Test fallback for XDG_CONFIG_HOME
+        with patch.dict("os.environ"):
+            os.environ.pop("XDG_CONFIG_HOME", None)
+            fallback_user_config = Path("~/.config").expanduser() / "black"
+            self.assertEqual(
+                black.files.find_user_pyproject_toml(), fallback_user_config.resolve()
             )
-            self.assertEqual(black.find_project_root((src_dir,)), src_dir.resolve())
-            self.assertEqual(black.find_project_root((src_python,)), src_dir.resolve())
+
+    def test_find_user_pyproject_toml_windows(self) -> None:
+        if system() != "Windows":
+            return
+
+        user_config_path = Path.home() / ".black"
+        self.assertEqual(
+            black.files.find_user_pyproject_toml(), user_config_path.resolve()
+        )
 
     def test_bpo_33660_workaround(self) -> None:
         if system() == "Windows":
             return
 
         # https://bugs.python.org/issue33660
-
-        old_cwd = Path.cwd()
-        try:
-            root = Path("/")
-            os.chdir(str(root))
+        root = Path("/")
+        with change_directory(root):
             path = Path("workspace") / "project"
             report = black.Report(verbose=True)
             normalized_path = black.normalize_path_maybe_ignore(path, root, report)
             self.assertEqual(normalized_path, "workspace/project")
+
+    def test_normalize_path_ignore_windows_junctions_outside_of_root(self) -> None:
+        if system() != "Windows":
+            return
+
+        with TemporaryDirectory() as workspace:
+            root = Path(workspace)
+            junction_dir = root / "junction"
+            junction_target_outside_of_root = root / ".."
+            os.system(f"mklink /J {junction_dir} {junction_target_outside_of_root}")
+
+            report = black.Report(verbose=True)
+            normalized_path = black.normalize_path_maybe_ignore(
+                junction_dir, root, report
+            )
+            # Manually delete for Python < 3.8
+            os.system(f"rmdir {junction_dir}")
+
+            self.assertEqual(normalized_path, None)
+
+    def test_newline_comment_interaction(self) -> None:
+        source = "class A:\\\r\n# type: ignore\n pass\n"
+        output = black.format_str(source, mode=DEFAULT_MODE)
+        black.assert_stable(source, output, mode=DEFAULT_MODE)
+
+    def test_bpo_2142_workaround(self) -> None:
+
+        # https://bugs.python.org/issue2142
+
+        source, _ = read_data("missing_final_newline.py")
+        # read_data adds a trailing newline
+        source = source.rstrip()
+        expected, _ = read_data("missing_final_newline.diff")
+        tmp_file = Path(black.dump_to_file(source, ensure_final_newline=False))
+        diff_header = re.compile(
+            rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d "
+            r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d"
+        )
+        try:
+            result = BlackRunner().invoke(black.main, ["--diff", str(tmp_file)])
+            self.assertEqual(result.exit_code, 0)
         finally:
-            os.chdir(str(old_cwd))
-
-
-class BlackDTestCase(AioHTTPTestCase):
-    async def get_application(self) -> web.Application:
-        return blackd.make_app()
-
-    # TODO: remove these decorators once the below is released
-    # https://github.com/aio-libs/aiohttp/pull/3727
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_request_needs_formatting(self) -> None:
-        response = await self.client.post("/", data=b"print('hello world')")
-        self.assertEqual(response.status, 200)
-        self.assertEqual(response.charset, "utf8")
-        self.assertEqual(await response.read(), b'print("hello world")\n')
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_request_no_change(self) -> None:
-        response = await self.client.post("/", data=b'print("hello world")\n')
-        self.assertEqual(response.status, 204)
-        self.assertEqual(await response.read(), b"")
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_request_syntax_error(self) -> None:
-        response = await self.client.post("/", data=b"what even ( is")
-        self.assertEqual(response.status, 400)
-        content = await response.text()
-        self.assertTrue(
-            content.startswith("Cannot parse"),
-            msg=f"Expected error to start with 'Cannot parse', got {repr(content)}",
+            os.unlink(tmp_file)
+        actual = result.output
+        actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
+        self.assertEqual(actual, expected)
+
+    @staticmethod
+    def compare_results(
+        result: click.testing.Result, expected_value: str, expected_exit_code: int
+    ) -> None:
+        """Helper method to test the value and exit code of a click Result."""
+        assert (
+            result.output == expected_value
+        ), "The output did not match the expected value."
+        assert result.exit_code == expected_exit_code, "The exit code is incorrect."
+
+    def test_code_option(self) -> None:
+        """Test the code option with no changes."""
+        code = 'print("Hello world")\n'
+        args = ["--code", code]
+        result = CliRunner().invoke(black.main, args)
+
+        self.compare_results(result, code, 0)
+
+    def test_code_option_changed(self) -> None:
+        """Test the code option when changes are required."""
+        code = "print('hello world')"
+        formatted = black.format_str(code, mode=DEFAULT_MODE)
+
+        args = ["--code", code]
+        result = CliRunner().invoke(black.main, args)
+
+        self.compare_results(result, formatted, 0)
+
+    def test_code_option_check(self) -> None:
+        """Test the code option when check is passed."""
+        args = ["--check", "--code", 'print("Hello world")\n']
+        result = CliRunner().invoke(black.main, args)
+        self.compare_results(result, "", 0)
+
+    def test_code_option_check_changed(self) -> None:
+        """Test the code option when changes are required, and check is passed."""
+        args = ["--check", "--code", "print('hello world')"]
+        result = CliRunner().invoke(black.main, args)
+        self.compare_results(result, "", 1)
+
+    def test_code_option_diff(self) -> None:
+        """Test the code option when diff is passed."""
+        code = "print('hello world')"
+        formatted = black.format_str(code, mode=DEFAULT_MODE)
+        result_diff = diff(code, formatted, "STDIN", "STDOUT")
+
+        args = ["--diff", "--code", code]
+        result = CliRunner().invoke(black.main, args)
+
+        # Remove time from diff
+        output = DIFF_TIME.sub("", result.output)
+
+        assert output == result_diff, "The output did not match the expected value."
+        assert result.exit_code == 0, "The exit code is incorrect."
+
+    def test_code_option_color_diff(self) -> None:
+        """Test the code option when color and diff are passed."""
+        code = "print('hello world')"
+        formatted = black.format_str(code, mode=DEFAULT_MODE)
+
+        result_diff = diff(code, formatted, "STDIN", "STDOUT")
+        result_diff = color_diff(result_diff)
+
+        args = ["--diff", "--color", "--code", code]
+        result = CliRunner().invoke(black.main, args)
+
+        # Remove time from diff
+        output = DIFF_TIME.sub("", result.output)
+
+        assert output == result_diff, "The output did not match the expected value."
+        assert result.exit_code == 0, "The exit code is incorrect."
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_code_option_safe(self) -> None:
+        """Test that the code option throws an error when the sanity checks fail."""
+        # Patch black.assert_equivalent to ensure the sanity checks fail
+        with patch.object(black, "assert_equivalent", side_effect=AssertionError):
+            code = 'print("Hello world")'
+            error_msg = f"{code}\nerror: cannot format <string>: \n"
+
+            args = ["--safe", "--code", code]
+            result = CliRunner().invoke(black.main, args)
+
+            self.compare_results(result, error_msg, 123)
+
+    def test_code_option_fast(self) -> None:
+        """Test that the code option ignores errors when the sanity checks fail."""
+        # Patch black.assert_equivalent to ensure the sanity checks fail
+        with patch.object(black, "assert_equivalent", side_effect=AssertionError):
+            code = 'print("Hello world")'
+            formatted = black.format_str(code, mode=DEFAULT_MODE)
+
+            args = ["--fast", "--code", code]
+            result = CliRunner().invoke(black.main, args)
+
+            self.compare_results(result, formatted, 0)
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_code_option_config(self) -> None:
+        """
+        Test that the code option finds the pyproject.toml in the current directory.
+        """
+        with patch.object(black, "parse_pyproject_toml", return_value={}) as parse:
+            args = ["--code", "print"]
+            # This is the only directory known to contain a pyproject.toml
+            with change_directory(PROJECT_ROOT):
+                CliRunner().invoke(black.main, args)
+                pyproject_path = Path(Path.cwd(), "pyproject.toml").resolve()
+
+            assert (
+                len(parse.mock_calls) >= 1
+            ), "Expected config parse to be called with the current directory."
+
+            _, call_args, _ = parse.mock_calls[0]
+            assert (
+                call_args[0].lower() == str(pyproject_path).lower()
+            ), "Incorrect config loaded."
+
+    @pytest.mark.incompatible_with_mypyc
+    def test_code_option_parent_config(self) -> None:
+        """
+        Test that the code option finds the pyproject.toml in the parent directory.
+        """
+        with patch.object(black, "parse_pyproject_toml", return_value={}) as parse:
+            with change_directory(THIS_DIR):
+                args = ["--code", "print"]
+                CliRunner().invoke(black.main, args)
+
+                pyproject_path = Path(Path().cwd().parent, "pyproject.toml").resolve()
+                assert (
+                    len(parse.mock_calls) >= 1
+                ), "Expected config parse to be called with the current directory."
+
+                _, call_args, _ = parse.mock_calls[0]
+                assert (
+                    call_args[0].lower() == str(pyproject_path).lower()
+                ), "Incorrect config loaded."
+
+    def test_for_handled_unexpected_eof_error(self) -> None:
+        """
+        Test that an unexpected EOF SyntaxError is nicely presented.
+        """
+        with pytest.raises(black.parsing.InvalidInput) as exc_info:
+            black.lib2to3_parse("print(", {})
+
+        exc_info.match("Cannot parse: 2:0: EOF in multi-line statement")
+
+    def test_equivalency_ast_parse_failure_includes_error(self) -> None:
+        with pytest.raises(AssertionError) as err:
+            black.assert_equivalent("a«»a  = 1", "a«»a  = 1")
+
+        err.match("--safe")
+        # Unfortunately the SyntaxError message has changed in newer versions so we
+        # can't match it directly.
+        err.match("invalid character")
+        err.match(r"\(<unknown>, line 1\)")
+
+
+class TestCaching:
+    def test_get_cache_dir(
+        self,
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        # Create multiple cache directories
+        workspace1 = tmp_path / "ws1"
+        workspace1.mkdir()
+        workspace2 = tmp_path / "ws2"
+        workspace2.mkdir()
+
+        # Force user_cache_dir to use the temporary directory for easier assertions
+        patch_user_cache_dir = patch(
+            target="black.cache.user_cache_dir",
+            autospec=True,
+            return_value=str(workspace1),
+        )
+
+        # If BLACK_CACHE_DIR is not set, use user_cache_dir
+        monkeypatch.delenv("BLACK_CACHE_DIR", raising=False)
+        with patch_user_cache_dir:
+            assert get_cache_dir() == workspace1
+
+        # If it is set, use the path provided in the env var.
+        monkeypatch.setenv("BLACK_CACHE_DIR", str(workspace2))
+        assert get_cache_dir() == workspace2
+
+    def test_cache_broken_file(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace:
+            cache_file = get_cache_file(mode)
+            cache_file.write_text("this is not a pickle")
+            assert black.read_cache(mode) == {}
+            src = (workspace / "test.py").resolve()
+            src.write_text("print('hello')")
+            invokeBlack([str(src)])
+            cache = black.read_cache(mode)
+            assert str(src) in cache
+
+    def test_cache_single_file_already_cached(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace:
+            src = (workspace / "test.py").resolve()
+            src.write_text("print('hello')")
+            black.write_cache({}, [src], mode)
+            invokeBlack([str(src)])
+            assert src.read_text() == "print('hello')"
+
+    @event_loop()
+    def test_cache_multiple_files(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace, patch(
+            "black.ProcessPoolExecutor", new=ThreadPoolExecutor
+        ):
+            one = (workspace / "one.py").resolve()
+            with one.open("w") as fobj:
+                fobj.write("print('hello')")
+            two = (workspace / "two.py").resolve()
+            with two.open("w") as fobj:
+                fobj.write("print('hello')")
+            black.write_cache({}, [one], mode)
+            invokeBlack([str(workspace)])
+            with one.open("r") as fobj:
+                assert fobj.read() == "print('hello')"
+            with two.open("r") as fobj:
+                assert fobj.read() == 'print("hello")\n'
+            cache = black.read_cache(mode)
+            assert str(one) in cache
+            assert str(two) in cache
+
+    @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"])
+    def test_no_cache_when_writeback_diff(self, color: bool) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace:
+            src = (workspace / "test.py").resolve()
+            with src.open("w") as fobj:
+                fobj.write("print('hello')")
+            with patch("black.read_cache") as read_cache, patch(
+                "black.write_cache"
+            ) as write_cache:
+                cmd = [str(src), "--diff"]
+                if color:
+                    cmd.append("--color")
+                invokeBlack(cmd)
+                cache_file = get_cache_file(mode)
+                assert cache_file.exists() is False
+                write_cache.assert_not_called()
+                read_cache.assert_not_called()
+
+    @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"])
+    @event_loop()
+    def test_output_locking_when_writeback_diff(self, color: bool) -> None:
+        with cache_dir() as workspace:
+            for tag in range(0, 4):
+                src = (workspace / f"test{tag}.py").resolve()
+                with src.open("w") as fobj:
+                    fobj.write("print('hello')")
+            with patch("black.Manager", wraps=multiprocessing.Manager) as mgr:
+                cmd = ["--diff", str(workspace)]
+                if color:
+                    cmd.append("--color")
+                invokeBlack(cmd, exit_code=0)
+                # this isn't quite doing what we want, but if it _isn't_
+                # called then we cannot be using the lock it provides
+                mgr.assert_called()
+
+    def test_no_cache_when_stdin(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir():
+            result = CliRunner().invoke(
+                black.main, ["-"], input=BytesIO(b"print('hello')")
+            )
+            assert not result.exit_code
+            cache_file = get_cache_file(mode)
+            assert not cache_file.exists()
+
+    def test_read_cache_no_cachefile(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir():
+            assert black.read_cache(mode) == {}
+
+    def test_write_cache_read_cache(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace:
+            src = (workspace / "test.py").resolve()
+            src.touch()
+            black.write_cache({}, [src], mode)
+            cache = black.read_cache(mode)
+            assert str(src) in cache
+            assert cache[str(src)] == black.get_cache_info(src)
+
+    def test_filter_cached(self) -> None:
+        with TemporaryDirectory() as workspace:
+            path = Path(workspace)
+            uncached = (path / "uncached").resolve()
+            cached = (path / "cached").resolve()
+            cached_but_changed = (path / "changed").resolve()
+            uncached.touch()
+            cached.touch()
+            cached_but_changed.touch()
+            cache = {
+                str(cached): black.get_cache_info(cached),
+                str(cached_but_changed): (0.0, 0),
+            }
+            todo, done = black.filter_cached(
+                cache, {uncached, cached, cached_but_changed}
+            )
+            assert todo == {uncached, cached_but_changed}
+            assert done == {cached}
+
+    def test_write_cache_creates_directory_if_needed(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir(exists=False) as workspace:
+            assert not workspace.exists()
+            black.write_cache({}, [], mode)
+            assert workspace.exists()
+
+    @event_loop()
+    def test_failed_formatting_does_not_get_cached(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir() as workspace, patch(
+            "black.ProcessPoolExecutor", new=ThreadPoolExecutor
+        ):
+            failing = (workspace / "failing.py").resolve()
+            with failing.open("w") as fobj:
+                fobj.write("not actually python")
+            clean = (workspace / "clean.py").resolve()
+            with clean.open("w") as fobj:
+                fobj.write('print("hello")\n')
+            invokeBlack([str(workspace)], exit_code=123)
+            cache = black.read_cache(mode)
+            assert str(failing) not in cache
+            assert str(clean) in cache
+
+    def test_write_cache_write_fail(self) -> None:
+        mode = DEFAULT_MODE
+        with cache_dir(), patch.object(Path, "open") as mock:
+            mock.side_effect = OSError
+            black.write_cache({}, [], mode)
+
+    def test_read_cache_line_lengths(self) -> None:
+        mode = DEFAULT_MODE
+        short_mode = replace(DEFAULT_MODE, line_length=1)
+        with cache_dir() as workspace:
+            path = (workspace / "file.py").resolve()
+            path.touch()
+            black.write_cache({}, [path], mode)
+            one = black.read_cache(mode)
+            assert str(path) in one
+            two = black.read_cache(short_mode)
+            assert str(path) not in two
+
+
+def assert_collected_sources(
+    src: Sequence[Union[str, Path]],
+    expected: Sequence[Union[str, Path]],
+    *,
+    ctx: Optional[FakeContext] = None,
+    exclude: Optional[str] = None,
+    include: Optional[str] = None,
+    extend_exclude: Optional[str] = None,
+    force_exclude: Optional[str] = None,
+    stdin_filename: Optional[str] = None,
+) -> None:
+    gs_src = tuple(str(Path(s)) for s in src)
+    gs_expected = [Path(s) for s in expected]
+    gs_exclude = None if exclude is None else compile_pattern(exclude)
+    gs_include = DEFAULT_INCLUDE if include is None else compile_pattern(include)
+    gs_extend_exclude = (
+        None if extend_exclude is None else compile_pattern(extend_exclude)
+    )
+    gs_force_exclude = None if force_exclude is None else compile_pattern(force_exclude)
+    collected = black.get_sources(
+        ctx=ctx or FakeContext(),
+        src=gs_src,
+        quiet=False,
+        verbose=False,
+        include=gs_include,
+        exclude=gs_exclude,
+        extend_exclude=gs_extend_exclude,
+        force_exclude=gs_force_exclude,
+        report=black.Report(),
+        stdin_filename=stdin_filename,
+    )
+    assert sorted(collected) == sorted(gs_expected)
+
+
+class TestFileCollection:
+    def test_include_exclude(self) -> None:
+        path = THIS_DIR / "data" / "include_exclude_tests"
+        src = [path]
+        expected = [
+            Path(path / "b/dont_exclude/a.py"),
+            Path(path / "b/dont_exclude/a.pyi"),
+        ]
+        assert_collected_sources(
+            src,
+            expected,
+            include=r"\.pyi?$",
+            exclude=r"/exclude/|/\.definitely_exclude/",
         )
 
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_unsupported_version(self) -> None:
-        response = await self.client.post(
-            "/", data=b"what", headers={blackd.PROTOCOL_VERSION_HEADER: "2"}
+    def test_gitignore_used_as_default(self) -> None:
+        base = Path(DATA_DIR / "include_exclude_tests")
+        expected = [
+            base / "b/.definitely_exclude/a.py",
+            base / "b/.definitely_exclude/a.pyi",
+        ]
+        src = [base / "b/"]
+        ctx = FakeContext()
+        ctx.obj["root"] = base
+        assert_collected_sources(src, expected, ctx=ctx, extend_exclude=r"/exclude/")
+
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_exclude_for_issue_1572(self) -> None:
+        # Exclude shouldn't touch files that were explicitly given to Black through the
+        # CLI. Exclude is supposed to only apply to the recursive discovery of files.
+        # https://github.com/psf/black/issues/1572
+        path = DATA_DIR / "include_exclude_tests"
+        src = [path / "b/exclude/a.py"]
+        expected = [path / "b/exclude/a.py"]
+        assert_collected_sources(src, expected, include="", exclude=r"/exclude/|a\.py")
+
+    def test_gitignore_exclude(self) -> None:
+        path = THIS_DIR / "data" / "include_exclude_tests"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"")
+        report = black.Report()
+        gitignore = PathSpec.from_lines(
+            "gitwildmatch", ["exclude/", ".definitely_exclude"]
         )
-        self.assertEqual(response.status, 501)
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_supported_version(self) -> None:
-        response = await self.client.post(
-            "/", data=b"what", headers={blackd.PROTOCOL_VERSION_HEADER: "1"}
+        sources: List[Path] = []
+        expected = [
+            Path(path / "b/dont_exclude/a.py"),
+            Path(path / "b/dont_exclude/a.pyi"),
+        ]
+        this_abs = THIS_DIR.resolve()
+        sources.extend(
+            black.gen_python_files(
+                path.iterdir(),
+                this_abs,
+                include,
+                exclude,
+                None,
+                None,
+                report,
+                gitignore,
+                verbose=False,
+                quiet=False,
+            )
         )
-        self.assertEqual(response.status, 200)
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_invalid_python_variant(self) -> None:
-        async def check(header_value: str, expected_status: int = 400) -> None:
-            response = await self.client.post(
-                "/", data=b"what", headers={blackd.PYTHON_VARIANT_HEADER: header_value}
+        assert sorted(expected) == sorted(sources)
+
+    def test_nested_gitignore(self) -> None:
+        path = Path(THIS_DIR / "data" / "nested_gitignore_tests")
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"")
+        root_gitignore = black.files.get_gitignore(path)
+        report = black.Report()
+        expected: List[Path] = [
+            Path(path / "x.py"),
+            Path(path / "root/b.py"),
+            Path(path / "root/c.py"),
+            Path(path / "root/child/c.py"),
+        ]
+        this_abs = THIS_DIR.resolve()
+        sources = list(
+            black.gen_python_files(
+                path.iterdir(),
+                this_abs,
+                include,
+                exclude,
+                None,
+                None,
+                report,
+                root_gitignore,
+                verbose=False,
+                quiet=False,
             )
-            self.assertEqual(response.status, expected_status)
-
-        await check("lol")
-        await check("ruby3.5")
-        await check("pyi3.6")
-        await check("py1.5")
-        await check("2.8")
-        await check("py2.8")
-        await check("3.0")
-        await check("pypy3.0")
-        await check("jython3.4")
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_pyi(self) -> None:
-        source, expected = read_data("stub.pyi")
-        response = await self.client.post(
-            "/", data=source, headers={blackd.PYTHON_VARIANT_HEADER: "pyi"}
         )
-        self.assertEqual(response.status, 200)
-        self.assertEqual(await response.text(), expected)
+        assert sorted(expected) == sorted(sources)
 
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_diff(self) -> None:
-        diff_header = re.compile(
-            r"(In|Out)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d"
+    def test_invalid_gitignore(self) -> None:
+        path = THIS_DIR / "data" / "invalid_gitignore_tests"
+        empty_config = path / "pyproject.toml"
+        result = BlackRunner().invoke(
+            black.main, ["--verbose", "--config", str(empty_config), str(path)]
         )
+        assert result.exit_code == 1
+        assert result.stderr_bytes is not None
 
-        source, _ = read_data("blackd_diff.py")
-        expected, _ = read_data("blackd_diff.diff")
+        gitignore = path / ".gitignore"
+        assert f"Could not parse {gitignore}" in result.stderr_bytes.decode()
 
-        response = await self.client.post(
-            "/", data=source, headers={blackd.DIFF_HEADER: "true"}
+    def test_invalid_nested_gitignore(self) -> None:
+        path = THIS_DIR / "data" / "invalid_nested_gitignore_tests"
+        empty_config = path / "pyproject.toml"
+        result = BlackRunner().invoke(
+            black.main, ["--verbose", "--config", str(empty_config), str(path)]
         )
-        self.assertEqual(response.status, 200)
+        assert result.exit_code == 1
+        assert result.stderr_bytes is not None
 
-        actual = await response.text()
-        actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
-        self.assertEqual(actual, expected)
+        gitignore = path / "a" / ".gitignore"
+        assert f"Could not parse {gitignore}" in result.stderr_bytes.decode()
+
+    def test_empty_include(self) -> None:
+        path = DATA_DIR / "include_exclude_tests"
+        src = [path]
+        expected = [
+            Path(path / "b/exclude/a.pie"),
+            Path(path / "b/exclude/a.py"),
+            Path(path / "b/exclude/a.pyi"),
+            Path(path / "b/dont_exclude/a.pie"),
+            Path(path / "b/dont_exclude/a.py"),
+            Path(path / "b/dont_exclude/a.pyi"),
+            Path(path / "b/.definitely_exclude/a.pie"),
+            Path(path / "b/.definitely_exclude/a.py"),
+            Path(path / "b/.definitely_exclude/a.pyi"),
+            Path(path / ".gitignore"),
+            Path(path / "pyproject.toml"),
+        ]
+        # Setting exclude explicitly to an empty string to block .gitignore usage.
+        assert_collected_sources(src, expected, include="", exclude="")
 
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_python_variant(self) -> None:
-        code = (
-            "def f(\n"
-            "    and_has_a_bunch_of,\n"
-            "    very_long_arguments_too,\n"
-            "    and_lots_of_them_as_well_lol,\n"
-            "    **and_very_long_keyword_arguments\n"
-            "):\n"
-            "    pass\n"
+    def test_extend_exclude(self) -> None:
+        path = DATA_DIR / "include_exclude_tests"
+        src = [path]
+        expected = [
+            Path(path / "b/exclude/a.py"),
+            Path(path / "b/dont_exclude/a.py"),
+        ]
+        assert_collected_sources(
+            src, expected, exclude=r"\.pyi$", extend_exclude=r"\.definitely_exclude"
         )
 
-        async def check(header_value: str, expected_status: int) -> None:
-            response = await self.client.post(
-                "/", data=code, headers={blackd.PYTHON_VARIANT_HEADER: header_value}
-            )
-            self.assertEqual(
-                response.status, expected_status, msg=await response.text()
+    @pytest.mark.incompatible_with_mypyc
+    def test_symlink_out_of_root_directory(self) -> None:
+        path = MagicMock()
+        root = THIS_DIR.resolve()
+        child = MagicMock()
+        include = re.compile(black.DEFAULT_INCLUDES)
+        exclude = re.compile(black.DEFAULT_EXCLUDES)
+        report = black.Report()
+        gitignore = PathSpec.from_lines("gitwildmatch", [])
+        # `child` should behave like a symlink which resolved path is clearly
+        # outside of the `root` directory.
+        path.iterdir.return_value = [child]
+        child.resolve.return_value = Path("/a/b/c")
+        child.as_posix.return_value = "/a/b/c"
+        try:
+            list(
+                black.gen_python_files(
+                    path.iterdir(),
+                    root,
+                    include,
+                    exclude,
+                    None,
+                    None,
+                    report,
+                    gitignore,
+                    verbose=False,
+                    quiet=False,
+                )
             )
+        except ValueError as ve:
+            pytest.fail(f"`get_python_files_in_dir()` failed: {ve}")
+        path.iterdir.assert_called_once()
+        child.resolve.assert_called_once()
 
-        await check("3.6", 200)
-        await check("py3.6", 200)
-        await check("3.6,3.7", 200)
-        await check("3.6,py3.7", 200)
-        await check("py36,py37", 200)
-        await check("36", 200)
-        await check("3.6.4", 200)
-
-        await check("2", 204)
-        await check("2.7", 204)
-        await check("py2.7", 204)
-        await check("3.4", 204)
-        await check("py3.4", 204)
-        await check("py34,py36", 204)
-        await check("34", 204)
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_line_length(self) -> None:
-        response = await self.client.post(
-            "/", data=b'print("hello")\n', headers={blackd.LINE_LENGTH_HEADER: "7"}
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_get_sources_with_stdin(self) -> None:
+        src = ["-"]
+        expected = ["-"]
+        assert_collected_sources(src, expected, include="", exclude=r"/exclude/|a\.py")
+
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_get_sources_with_stdin_filename(self) -> None:
+        src = ["-"]
+        stdin_filename = str(THIS_DIR / "data/collections.py")
+        expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"]
+        assert_collected_sources(
+            src,
+            expected,
+            exclude=r"/exclude/a\.py",
+            stdin_filename=stdin_filename,
         )
-        self.assertEqual(response.status, 200)
-
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_invalid_line_length(self) -> None:
-        response = await self.client.post(
-            "/", data=b'print("hello")\n', headers={blackd.LINE_LENGTH_HEADER: "NaN"}
+
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_get_sources_with_stdin_filename_and_exclude(self) -> None:
+        # Exclude shouldn't exclude stdin_filename since it is mimicking the
+        # file being passed directly. This is the same as
+        # test_exclude_for_issue_1572
+        path = DATA_DIR / "include_exclude_tests"
+        src = ["-"]
+        stdin_filename = str(path / "b/exclude/a.py")
+        expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"]
+        assert_collected_sources(
+            src,
+            expected,
+            exclude=r"/exclude/|a\.py",
+            stdin_filename=stdin_filename,
         )
-        self.assertEqual(response.status, 400)
 
-    @skip_if_exception("ClientOSError")
-    @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed")
-    @unittest_run_loop
-    async def test_blackd_response_black_version_header(self) -> None:
-        response = await self.client.post("/")
-        self.assertIsNotNone(response.headers.get(blackd.BLACK_VERSION_HEADER))
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None:
+        # Extend exclude shouldn't exclude stdin_filename since it is mimicking the
+        # file being passed directly. This is the same as
+        # test_exclude_for_issue_1572
+        src = ["-"]
+        path = THIS_DIR / "data" / "include_exclude_tests"
+        stdin_filename = str(path / "b/exclude/a.py")
+        expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"]
+        assert_collected_sources(
+            src,
+            expected,
+            extend_exclude=r"/exclude/|a\.py",
+            stdin_filename=stdin_filename,
+        )
 
+    @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
+    def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None:
+        # Force exclude should exclude the file when passing it through
+        # stdin_filename
+        path = THIS_DIR / "data" / "include_exclude_tests"
+        stdin_filename = str(path / "b/exclude/a.py")
+        assert_collected_sources(
+            src=["-"],
+            expected=[],
+            force_exclude=r"/exclude/|a\.py",
+            stdin_filename=stdin_filename,
+        )
 
-with open(black.__file__, "r", encoding="utf-8") as _bf:
-    black_source_lines = _bf.readlines()
+
+try:
+    with open(black.__file__, "r", encoding="utf-8") as _bf:
+        black_source_lines = _bf.readlines()
+except UnicodeDecodeError:
+    if not black.COMPILED:
+        raise
 
 
-def tracefunc(frame: types.FrameType, event: str, arg: Any) -> Callable:
+def tracefunc(
+    frame: types.FrameType, event: str, arg: Any
+) -> Callable[[types.FrameType, str, Any], Any]:
     """Show function calls `from black/__init__.py` as they happen.
 
     Register this with `sys.settrace()` in a test you're debugging.
@@ -2245,7 +2127,3 @@ def tracefunc(frame: types.FrameType, event: str, arg: Any) -> Callable:
     if "black/__init__.py" in filename:
         print(f"{' ' * stack}{lineno}:{funcname}")
     return tracefunc
-
-
-if __name__ == "__main__":
-    unittest.main(module="test_black")