Added --include and --exclude cli options (#281)

These 2 options allow you to pass in regular expressions that determine
whether files/directories are included or excluded in the recursive file
search.

Fixes #270

diff --git a/README.md b/README.md
index 12af03f4d7d391e226c57bbfdba1baa47aecd863..79b83476c1ecc7816b0bb75fac9ef935d41aec3f 100644 (file)
--- a/README.md
+++ b/README.md
@@ -89,6 +89,17 @@ Options:
                               **kwargs.  [default: per-file auto-detection]
   -S, --skip-string-normalization
                               Don't normalize string quotes or prefixes.
                               **kwargs.  [default: per-file auto-detection]
   -S, --skip-string-normalization
                               Don't normalize string quotes or prefixes.

+  --include TEXT              A regular expression that matches files and
+                              directories that should be included on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default: \.pyi?$]
+  --exclude TEXT              A regular expression that matches files and
+                              directories that should be excluded on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default:
+                              build/|buck-out/|dist/|_build/|\.git/|\.hg/|
+                              \.mypy_cache/|\.tox/|\.venv/]
+

   --version                   Show the version and exit.
   --help                      Show this message and exit.
 ```
   --version                   Show the version and exit.
   --help                      Show this message and exit.
 ```


@@ -698,6 +709,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.6b0
 
 
 ### 18.6b0
 

+* added `--include` and `--exclude` (#270)
+

 * added `--skip-string-normalization` (#118)
 
 * fixed stdin handling not working correctly if an old version of Click was
 * added `--skip-string-normalization` (#118)
 
 * fixed stdin handling not working correctly if an old version of Click was

diff --git a/black.py b/black.py
index 4599bdd98a9ae536bb25c27d4b3d1511e0523f17..ce4a44f392f3daadf1e1f4be88aded372a6c7d96 100644 (file)
--- a/black.py
+++ b/black.py
@@ -46,6 +46,10 @@ from blib2to3.pgen2.parse import ParseError
 
 __version__ = "18.5b1"
 DEFAULT_LINE_LENGTH = 88
 
 __version__ = "18.5b1"
 DEFAULT_LINE_LENGTH = 88

+DEFAULT_EXCLUDES = (
+    r"build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/"
+)
+DEFAULT_INCLUDES = r"\.pyi?$"

 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 
 
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 
 


@@ -189,6 +193,28 @@ class FileMode(Flag):
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )

+@click.option(
+    "--include",
+    type=str,
+    default=DEFAULT_INCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "included on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)
+@click.option(
+    "--exclude",
+    type=str,
+    default=DEFAULT_EXCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "excluded on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)

 @click.version_option(version=__version__)
 @click.argument(
     "src",
 @click.version_option(version=__version__)
 @click.argument(
     "src",


@@ -208,14 +234,26 @@ def main(
     py36: bool,
     skip_string_normalization: bool,
     quiet: bool,
     py36: bool,
     skip_string_normalization: bool,
     quiet: bool,

+    include: str,
+    exclude: str,

     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []
     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []

+    try:
+        include_regex = re.compile(include)
+    except re.error:
+        err(f"Invalid regular expression for include given: {include!r}")
+        ctx.exit(2)
+    try:
+        exclude_regex = re.compile(exclude)
+    except re.error:
+        err(f"Invalid regular expression for exclude given: {exclude!r}")
+        ctx.exit(2)

     for s in src:
         p = Path(s)
         if p.is_dir():
     for s in src:
         p = Path(s)
         if p.is_dir():

-            sources.extend(gen_python_files_in_dir(p))
+            sources.extend(gen_python_files_in_dir(p, include_regex, exclude_regex))

         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)
         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)


@@ -2750,33 +2788,35 @@ def get_future_imports(node: Node) -> Set[str]:
     return imports
 
 
     return imports
 
 

-PYTHON_EXTENSIONS = {".py", ".pyi"}
-BLACKLISTED_DIRECTORIES = {
-    "build",
-    "buck-out",
-    "dist",
-    "_build",
-    ".git",
-    ".hg",
-    ".mypy_cache",
-    ".tox",
-    ".venv",
-}
-
-
-def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
-    """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
-    and have one of the PYTHON_EXTENSIONS.
+def gen_python_files_in_dir(
+    path: Path, include: Pattern[str], exclude: Pattern[str]
+) -> Iterator[Path]:
+    """Generate all files under `path` whose paths are not excluded by the
+    `exclude` regex, but are included by the `include` regex.

     """
     """

+

     for child in path.iterdir():
     for child in path.iterdir():

+        searchable_path = str(child.as_posix())
+        if Path(child.parts[0]).is_dir():
+            searchable_path = "/" + searchable_path

         if child.is_dir():
         if child.is_dir():

-            if child.name in BLACKLISTED_DIRECTORIES:
+            searchable_path = searchable_path + "/"
+            exclude_match = exclude.search(searchable_path)
+            if exclude_match and len(exclude_match.group()) > 0:

                 continue
 
                 continue
 

-            yield from gen_python_files_in_dir(child)
+            yield from gen_python_files_in_dir(child, include, exclude)

 
 

-        elif child.is_file() and child.suffix in PYTHON_EXTENSIONS:
-            yield child
+        else:
+            include_match = include.search(searchable_path)
+            exclude_match = exclude.search(searchable_path)
+            if (
+                child.is_file()
+                and include_match
+                and len(include_match.group()) > 0
+                and (not exclude_match or len(exclude_match.group()) == 0)
+            ):
+                yield child

 
 
 @dataclass
 
 
 @dataclass

diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.pie b/tests/include_exclude_tests/b/.definitely_exclude/a.pie
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.py b/tests/include_exclude_tests/b/.definitely_exclude/a.py
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.pyi b/tests/include_exclude_tests/b/.definitely_exclude/a.pyi
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/dont_exclude/a.pie b/tests/include_exclude_tests/b/dont_exclude/a.pie
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/dont_exclude/a.py b/tests/include_exclude_tests/b/dont_exclude/a.py
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/dont_exclude/a.pyi b/tests/include_exclude_tests/b/dont_exclude/a.pyi
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/exclude/a.pie b/tests/include_exclude_tests/b/exclude/a.pie
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/exclude/a.py b/tests/include_exclude_tests/b/exclude/a.py
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/include_exclude_tests/b/exclude/a.pyi b/tests/include_exclude_tests/b/exclude/a.pyi
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/tests/test_black.py b/tests/test_black.py
index e654d0d532f91dcacd7a4fcfcd278e8aea61eb65..c10dd1d90c3d3d217895ea52ba644010913a5ce6 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -11,6 +11,7 @@ from tempfile import TemporaryDirectory
 from typing import Any, List, Tuple, Iterator
 import unittest
 from unittest.mock import patch
 from typing import Any, List, Tuple, Iterator
 import unittest
 from unittest.mock import patch

+import re

 
 from click import unstyle
 from click.testing import CliRunner
 
 from click import unstyle
 from click.testing import CliRunner


@@ -851,6 +852,53 @@ class BlackTestCase(unittest.TestCase):
         actual = result.output
         self.assertFormatEqual(actual, expected)
 
         actual = result.output
         self.assertFormatEqual(actual, expected)
 

+    def test_include_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"/exclude/|/\.definitely_exclude/")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+        ]
+        sources.extend(black.gen_python_files_in_dir(path, include, exclude))
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_empty_include(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, empty, re.compile(black.DEFAULT_EXCLUDES)
+            )
+        )
+        self.assertEqual([], (sources))
+
+    def test_empty_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.pyi"),
+        ]
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, re.compile(black.DEFAULT_INCLUDES), empty
+            )
+        )
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_invalid_include_exclude(self) -> None:
+        for option in ["--include", "--exclude"]:
+            result = CliRunner().invoke(black.main, ["-", option, "**()(!!*)"])
+            self.assertEqual(result.exit_code, 2)
+

 
 if __name__ == "__main__":
     unittest.main()
 
 if __name__ == "__main__":
     unittest.main()