]> git.madduck.net Git - etc/vim.git/blob - src/black/files.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

65b2d0a840255fd245c303e69e7a07f12f16b386
[etc/vim.git] / src / black / files.py
1 import io
2 import os
3 import sys
4 from functools import lru_cache
5 from pathlib import Path
6 from typing import (
7     TYPE_CHECKING,
8     Any,
9     Dict,
10     Iterable,
11     Iterator,
12     List,
13     Optional,
14     Pattern,
15     Sequence,
16     Tuple,
17     Union,
18 )
19
20 from mypy_extensions import mypyc_attr
21 from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
22 from packaging.version import InvalidVersion, Version
23 from pathspec import PathSpec
24 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
25
26 if sys.version_info >= (3, 11):
27     try:
28         import tomllib
29     except ImportError:
30         # Help users on older alphas
31         if not TYPE_CHECKING:
32             import tomli as tomllib
33 else:
34     import tomli as tomllib
35
36 from black.handle_ipynb_magics import jupyter_dependencies_are_installed
37 from black.mode import TargetVersion
38 from black.output import err
39 from black.report import Report
40
41 if TYPE_CHECKING:
42     import colorama  # noqa: F401
43
44
45 @lru_cache()
46 def find_project_root(
47     srcs: Sequence[str], stdin_filename: Optional[str] = None
48 ) -> Tuple[Path, str]:
49     """Return a directory containing .git, .hg, or pyproject.toml.
50
51     That directory will be a common parent of all files and directories
52     passed in `srcs`.
53
54     If no directory in the tree contains a marker that would specify it's the
55     project root, the root of the file system is returned.
56
57     Returns a two-tuple with the first element as the project root path and
58     the second element as a string describing the method by which the
59     project root was discovered.
60     """
61     if stdin_filename is not None:
62         srcs = tuple(stdin_filename if s == "-" else s for s in srcs)
63     if not srcs:
64         srcs = [str(Path.cwd().resolve())]
65
66     path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs]
67
68     # A list of lists of parents for each 'src'. 'src' is included as a
69     # "parent" of itself if it is a directory
70     src_parents = [
71         list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs
72     ]
73
74     common_base = max(
75         set.intersection(*(set(parents) for parents in src_parents)),
76         key=lambda path: path.parts,
77     )
78
79     for directory in (common_base, *common_base.parents):
80         if (directory / ".git").exists():
81             return directory, ".git directory"
82
83         if (directory / ".hg").is_dir():
84             return directory, ".hg directory"
85
86         if (directory / "pyproject.toml").is_file():
87             return directory, "pyproject.toml"
88
89     return directory, "file system root"
90
91
92 def find_pyproject_toml(
93     path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None
94 ) -> Optional[str]:
95     """Find the absolute filepath to a pyproject.toml if it exists"""
96     path_project_root, _ = find_project_root(path_search_start, stdin_filename)
97     path_pyproject_toml = path_project_root / "pyproject.toml"
98     if path_pyproject_toml.is_file():
99         return str(path_pyproject_toml)
100
101     try:
102         path_user_pyproject_toml = find_user_pyproject_toml()
103         return (
104             str(path_user_pyproject_toml)
105             if path_user_pyproject_toml.is_file()
106             else None
107         )
108     except (PermissionError, RuntimeError) as e:
109         # We do not have access to the user-level config directory, so ignore it.
110         err(f"Ignoring user configuration directory due to {e!r}")
111         return None
112
113
114 @mypyc_attr(patchable=True)
115 def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
116     """Parse a pyproject toml file, pulling out relevant parts for Black.
117
118     If parsing fails, will raise a tomllib.TOMLDecodeError.
119     """
120     with open(path_config, "rb") as f:
121         pyproject_toml = tomllib.load(f)
122     config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {})
123     config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
124
125     if "target_version" not in config:
126         inferred_target_version = infer_target_version(pyproject_toml)
127         if inferred_target_version is not None:
128             config["target_version"] = [v.name.lower() for v in inferred_target_version]
129
130     return config
131
132
133 def infer_target_version(
134     pyproject_toml: Dict[str, Any]
135 ) -> Optional[List[TargetVersion]]:
136     """Infer Black's target version from the project metadata in pyproject.toml.
137
138     Supports the PyPA standard format (PEP 621):
139     https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#requires-python
140
141     If the target version cannot be inferred, returns None.
142     """
143     project_metadata = pyproject_toml.get("project", {})
144     requires_python = project_metadata.get("requires-python", None)
145     if requires_python is not None:
146         try:
147             return parse_req_python_version(requires_python)
148         except InvalidVersion:
149             pass
150         try:
151             return parse_req_python_specifier(requires_python)
152         except (InvalidSpecifier, InvalidVersion):
153             pass
154
155     return None
156
157
158 def parse_req_python_version(requires_python: str) -> Optional[List[TargetVersion]]:
159     """Parse a version string (i.e. ``"3.7"``) to a list of TargetVersion.
160
161     If parsing fails, will raise a packaging.version.InvalidVersion error.
162     If the parsed version cannot be mapped to a valid TargetVersion, returns None.
163     """
164     version = Version(requires_python)
165     if version.release[0] != 3:
166         return None
167     try:
168         return [TargetVersion(version.release[1])]
169     except (IndexError, ValueError):
170         return None
171
172
173 def parse_req_python_specifier(requires_python: str) -> Optional[List[TargetVersion]]:
174     """Parse a specifier string (i.e. ``">=3.7,<3.10"``) to a list of TargetVersion.
175
176     If parsing fails, will raise a packaging.specifiers.InvalidSpecifier error.
177     If the parsed specifier cannot be mapped to a valid TargetVersion, returns None.
178     """
179     specifier_set = strip_specifier_set(SpecifierSet(requires_python))
180     if not specifier_set:
181         return None
182
183     target_version_map = {f"3.{v.value}": v for v in TargetVersion}
184     compatible_versions: List[str] = list(specifier_set.filter(target_version_map))
185     if compatible_versions:
186         return [target_version_map[v] for v in compatible_versions]
187     return None
188
189
190 def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet:
191     """Strip minor versions for some specifiers in the specifier set.
192
193     For background on version specifiers, see PEP 440:
194     https://peps.python.org/pep-0440/#version-specifiers
195     """
196     specifiers = []
197     for s in specifier_set:
198         if "*" in str(s):
199             specifiers.append(s)
200         elif s.operator in ["~=", "==", ">=", "==="]:
201             version = Version(s.version)
202             stripped = Specifier(f"{s.operator}{version.major}.{version.minor}")
203             specifiers.append(stripped)
204         elif s.operator == ">":
205             version = Version(s.version)
206             if len(version.release) > 2:
207                 s = Specifier(f">={version.major}.{version.minor}")
208             specifiers.append(s)
209         else:
210             specifiers.append(s)
211
212     return SpecifierSet(",".join(str(s) for s in specifiers))
213
214
215 @lru_cache()
216 def find_user_pyproject_toml() -> Path:
217     r"""Return the path to the top-level user configuration for black.
218
219     This looks for ~\.black on Windows and ~/.config/black on Linux and other
220     Unix systems.
221
222     May raise:
223     - RuntimeError: if the current user has no homedir
224     - PermissionError: if the current process cannot access the user's homedir
225     """
226     if sys.platform == "win32":
227         # Windows
228         user_config_path = Path.home() / ".black"
229     else:
230         config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config")
231         user_config_path = Path(config_root).expanduser() / "black"
232     return user_config_path.resolve()
233
234
235 @lru_cache()
236 def get_gitignore(root: Path) -> PathSpec:
237     """Return a PathSpec matching gitignore content if present."""
238     gitignore = root / ".gitignore"
239     lines: List[str] = []
240     if gitignore.is_file():
241         with gitignore.open(encoding="utf-8") as gf:
242             lines = gf.readlines()
243     try:
244         return PathSpec.from_lines("gitwildmatch", lines)
245     except GitWildMatchPatternError as e:
246         err(f"Could not parse {gitignore}: {e}")
247         raise
248
249
250 def normalize_path_maybe_ignore(
251     path: Path,
252     root: Path,
253     report: Optional[Report] = None,
254 ) -> Optional[str]:
255     """Normalize `path`. May return `None` if `path` was ignored.
256
257     `report` is where "path ignored" output goes.
258     """
259     try:
260         abspath = path if path.is_absolute() else Path.cwd() / path
261         normalized_path = abspath.resolve()
262         try:
263             root_relative_path = normalized_path.relative_to(root).as_posix()
264         except ValueError:
265             if report:
266                 report.path_ignored(
267                     path, f"is a symbolic link that points outside {root}"
268                 )
269             return None
270
271     except OSError as e:
272         if report:
273             report.path_ignored(path, f"cannot be read because {e}")
274         return None
275
276     return root_relative_path
277
278
279 def path_is_ignored(
280     path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report
281 ) -> bool:
282     for gitignore_path, pattern in gitignore_dict.items():
283         relative_path = normalize_path_maybe_ignore(path, gitignore_path, report)
284         if relative_path is None:
285             break
286         if pattern.match_file(relative_path):
287             report.path_ignored(path, "matches a .gitignore file content")
288             return True
289     return False
290
291
292 def path_is_excluded(
293     normalized_path: str,
294     pattern: Optional[Pattern[str]],
295 ) -> bool:
296     match = pattern.search(normalized_path) if pattern else None
297     return bool(match and match.group(0))
298
299
300 def gen_python_files(
301     paths: Iterable[Path],
302     root: Path,
303     include: Pattern[str],
304     exclude: Pattern[str],
305     extend_exclude: Optional[Pattern[str]],
306     force_exclude: Optional[Pattern[str]],
307     report: Report,
308     gitignore_dict: Optional[Dict[Path, PathSpec]],
309     *,
310     verbose: bool,
311     quiet: bool,
312 ) -> Iterator[Path]:
313     """Generate all files under `path` whose paths are not excluded by the
314     `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
315     but are included by the `include` regex.
316
317     Symbolic links pointing outside of the `root` directory are ignored.
318
319     `report` is where output about exclusions goes.
320     """
321
322     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
323     for child in paths:
324         normalized_path = normalize_path_maybe_ignore(child, root, report)
325         if normalized_path is None:
326             continue
327
328         # First ignore files matching .gitignore, if passed
329         if gitignore_dict and path_is_ignored(child, gitignore_dict, report):
330             continue
331
332         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
333         normalized_path = "/" + normalized_path
334         if child.is_dir():
335             normalized_path += "/"
336
337         if path_is_excluded(normalized_path, exclude):
338             report.path_ignored(child, "matches the --exclude regular expression")
339             continue
340
341         if path_is_excluded(normalized_path, extend_exclude):
342             report.path_ignored(
343                 child, "matches the --extend-exclude regular expression"
344             )
345             continue
346
347         if path_is_excluded(normalized_path, force_exclude):
348             report.path_ignored(child, "matches the --force-exclude regular expression")
349             continue
350
351         if child.is_dir():
352             # If gitignore is None, gitignore usage is disabled, while a Falsey
353             # gitignore is when the directory doesn't have a .gitignore file.
354             if gitignore_dict is not None:
355                 new_gitignore_dict = {
356                     **gitignore_dict,
357                     root / child: get_gitignore(child),
358                 }
359             else:
360                 new_gitignore_dict = None
361             yield from gen_python_files(
362                 child.iterdir(),
363                 root,
364                 include,
365                 exclude,
366                 extend_exclude,
367                 force_exclude,
368                 report,
369                 new_gitignore_dict,
370                 verbose=verbose,
371                 quiet=quiet,
372             )
373
374         elif child.is_file():
375             if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
376                 verbose=verbose, quiet=quiet
377             ):
378                 continue
379             include_match = include.search(normalized_path) if include else True
380             if include_match:
381                 yield child
382
383
384 def wrap_stream_for_windows(
385     f: io.TextIOWrapper,
386 ) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]:
387     """
388     Wrap stream with colorama's wrap_stream so colors are shown on Windows.
389
390     If `colorama` is unavailable, the original stream is returned unmodified.
391     Otherwise, the `wrap_stream()` function determines whether the stream needs
392     to be wrapped for a Windows environment and will accordingly either return
393     an `AnsiToWin32` wrapper or the original stream.
394     """
395     try:
396         from colorama.initialise import wrap_stream
397     except ImportError:
398         return f
399     else:
400         # Set `strip=False` to avoid needing to modify test_express_diff_with_color.
401         return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True)