From e9d4e7b67f359114ba57ecf0c1300f7eb4f08950 Mon Sep 17 00:00:00 2001 From: jgirardet Date: Mon, 21 Oct 2019 11:44:53 +0200 Subject: [PATCH 1/1] add gitignore support using pathspec (#878) --- Pipfile | 1 + black.py | 27 +++++++++++++++++++-- setup.py | 1 + tests/test_black.py | 57 +++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index c461004..fb52776 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ black = {path = ".",extras = ["d"],editable = true} aiohttp-cors = "*" typed-ast = ">=1.3.1" regex = ">=2019.8" +pathspec = ">=0.6" [dev-packages] pre-commit = "*" diff --git a/black.py b/black.py index 8a70c02..bf75bc0 100644 --- a/black.py +++ b/black.py @@ -43,6 +43,7 @@ from attr import dataclass, evolve, Factory import click import toml from typed_ast import ast3, ast27 +from pathspec import PathSpec # lib2to3 fork from blib2to3.pytree import Node, Leaf, type_repr @@ -436,7 +437,9 @@ def main( p = Path(s) if p.is_dir(): sources.update( - gen_python_files_in_dir(p, root, include_regex, exclude_regex, report) + gen_python_files_in_dir( + p, root, include_regex, exclude_regex, report, get_gitignore(root) + ) ) elif p.is_file() or s == "-": # if a file was explicitly given, we don't care about its extension @@ -3453,12 +3456,23 @@ def get_future_imports(node: Node) -> Set[str]: return imports +@lru_cache() +def get_gitignore(root: Path) -> PathSpec: + """ Return a PathSpec matching gitignore content if present.""" + gitignore = root / ".gitignore" + if not gitignore.is_file(): + return PathSpec.from_lines("gitwildmatch", []) + else: + return PathSpec.from_lines("gitwildmatch", gitignore.open()) + + def gen_python_files_in_dir( path: Path, root: Path, include: Pattern[str], exclude: Pattern[str], report: "Report", + gitignore: PathSpec, ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude` regex, but are included by the `include` regex. @@ -3469,6 +3483,12 @@ def gen_python_files_in_dir( """ assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in path.iterdir(): + # First ignore files matching .gitignore + if gitignore.match_file(child.as_posix()): + report.path_ignored(child, f"matches the .gitignore file content") + continue + + # Then ignore with `exclude` option. try: normalized_path = "/" + child.resolve().relative_to(root).as_posix() except ValueError: @@ -3482,13 +3502,16 @@ def gen_python_files_in_dir( if child.is_dir(): normalized_path += "/" + exclude_match = exclude.search(normalized_path) if exclude_match and exclude_match.group(0): report.path_ignored(child, f"matches the --exclude regular expression") continue if child.is_dir(): - yield from gen_python_files_in_dir(child, root, include, exclude, report) + yield from gen_python_files_in_dir( + child, root, include, exclude, report, gitignore + ) elif child.is_file(): include_match = include.search(normalized_path) diff --git a/setup.py b/setup.py index 8c9672e..37fe820 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ setup( "toml>=0.9.4", "typed-ast>=1.3.1", "regex", + "pathspec>=0.6, <1", ], extras_require={"d": ["aiohttp>=3.3.2", "aiohttp-cors"]}, test_suite="tests.test_black", diff --git a/tests/test_black.py b/tests/test_black.py index a92798d..e6a6647 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -29,6 +29,8 @@ except ImportError: else: has_blackd_deps = True +from pathspec import PathSpec + ff = partial(black.format_file_in_place, mode=black.FileMode(), fast=True) fs = partial(black.format_str, mode=black.FileMode()) THIS_FILE = Path(__file__) @@ -1392,6 +1394,7 @@ class BlackTestCase(unittest.TestCase): include = re.compile(r"\.pyi?$") exclude = re.compile(r"/exclude/|/\.definitely_exclude/") report = black.Report() + gitignore = PathSpec.from_lines("gitwildmatch", []) sources: List[Path] = [] expected = [ Path(path / "b/dont_exclude/a.py"), @@ -1399,13 +1402,37 @@ class BlackTestCase(unittest.TestCase): ] this_abs = THIS_DIR.resolve() sources.extend( - black.gen_python_files_in_dir(path, this_abs, include, exclude, report) + black.gen_python_files_in_dir( + path, this_abs, include, exclude, report, gitignore + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + + def test_gitignore_exclude(self) -> None: + path = THIS_DIR / "data" / "include_exclude_tests" + include = re.compile(r"\.pyi?$") + exclude = re.compile(r"") + report = black.Report() + gitignore = PathSpec.from_lines( + "gitwildmatch", ["exclude/", ".definitely_exclude"] + ) + sources: List[Path] = [] + expected = [ + Path(path / "b/dont_exclude/a.py"), + Path(path / "b/dont_exclude/a.pyi"), + ] + this_abs = THIS_DIR.resolve() + sources.extend( + black.gen_python_files_in_dir( + path, this_abs, include, exclude, report, gitignore + ) ) self.assertEqual(sorted(expected), sorted(sources)) def test_empty_include(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" report = black.Report() + gitignore = PathSpec.from_lines("gitwildmatch", []) empty = re.compile(r"") sources: List[Path] = [] expected = [ @@ -1422,7 +1449,12 @@ class BlackTestCase(unittest.TestCase): this_abs = THIS_DIR.resolve() sources.extend( black.gen_python_files_in_dir( - path, this_abs, empty, re.compile(black.DEFAULT_EXCLUDES), report + path, + this_abs, + empty, + re.compile(black.DEFAULT_EXCLUDES), + report, + gitignore, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1430,6 +1462,7 @@ class BlackTestCase(unittest.TestCase): def test_empty_exclude(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" report = black.Report() + gitignore = PathSpec.from_lines("gitwildmatch", []) empty = re.compile(r"") sources: List[Path] = [] expected = [ @@ -1443,7 +1476,12 @@ class BlackTestCase(unittest.TestCase): this_abs = THIS_DIR.resolve() sources.extend( black.gen_python_files_in_dir( - path, this_abs, re.compile(black.DEFAULT_INCLUDES), empty, report + path, + this_abs, + re.compile(black.DEFAULT_INCLUDES), + empty, + report, + gitignore, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1488,13 +1526,18 @@ class BlackTestCase(unittest.TestCase): include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() + gitignore = PathSpec.from_lines("gitwildmatch", []) # `child` should behave like a symlink which resolved path is clearly # outside of the `root` directory. path.iterdir.return_value = [child] child.resolve.return_value = Path("/a/b/c") child.is_symlink.return_value = True try: - list(black.gen_python_files_in_dir(path, root, include, exclude, report)) + list( + black.gen_python_files_in_dir( + path, root, include, exclude, report, gitignore + ) + ) except ValueError as ve: self.fail(f"`get_python_files_in_dir()` failed: {ve}") path.iterdir.assert_called_once() @@ -1504,7 +1547,11 @@ class BlackTestCase(unittest.TestCase): # outside of the `root` directory. child.is_symlink.return_value = False with self.assertRaises(ValueError): - list(black.gen_python_files_in_dir(path, root, include, exclude, report)) + list( + black.gen_python_files_in_dir( + path, root, include, exclude, report, gitignore + ) + ) path.iterdir.assert_called() self.assertEqual(path.iterdir.call_count, 2) child.resolve.assert_called() -- 2.39.5