]> git.madduck.net Git - etc/vim.git/blob - src/black/files.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Use GH action version when version argument not specified (#3543)
[etc/vim.git] / src / black / files.py
1 import io
2 import os
3 import sys
4 from functools import lru_cache
5 from pathlib import Path
6 from typing import (
7     TYPE_CHECKING,
8     Any,
9     Dict,
10     Iterable,
11     Iterator,
12     List,
13     Optional,
14     Pattern,
15     Sequence,
16     Tuple,
17     Union,
18 )
19
20 from mypy_extensions import mypyc_attr
21 from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
22 from packaging.version import InvalidVersion, Version
23 from pathspec import PathSpec
24 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
25
26 if sys.version_info >= (3, 11):
27     try:
28         import tomllib
29     except ImportError:
30         # Help users on older alphas
31         if not TYPE_CHECKING:
32             import tomli as tomllib
33 else:
34     import tomli as tomllib
35
36 from black.handle_ipynb_magics import jupyter_dependencies_are_installed
37 from black.mode import TargetVersion
38 from black.output import err
39 from black.report import Report
40
41 if TYPE_CHECKING:
42     import colorama  # noqa: F401
43
44
45 @lru_cache()
46 def find_project_root(
47     srcs: Sequence[str], stdin_filename: Optional[str] = None
48 ) -> Tuple[Path, str]:
49     """Return a directory containing .git, .hg, or pyproject.toml.
50
51     That directory will be a common parent of all files and directories
52     passed in `srcs`.
53
54     If no directory in the tree contains a marker that would specify it's the
55     project root, the root of the file system is returned.
56
57     Returns a two-tuple with the first element as the project root path and
58     the second element as a string describing the method by which the
59     project root was discovered.
60     """
61     if stdin_filename is not None:
62         srcs = tuple(stdin_filename if s == "-" else s for s in srcs)
63     if not srcs:
64         srcs = [str(Path.cwd().resolve())]
65
66     path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs]
67
68     # A list of lists of parents for each 'src'. 'src' is included as a
69     # "parent" of itself if it is a directory
70     src_parents = [
71         list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs
72     ]
73
74     common_base = max(
75         set.intersection(*(set(parents) for parents in src_parents)),
76         key=lambda path: path.parts,
77     )
78
79     for directory in (common_base, *common_base.parents):
80         if (directory / ".git").exists():
81             return directory, ".git directory"
82
83         if (directory / ".hg").is_dir():
84             return directory, ".hg directory"
85
86         if (directory / "pyproject.toml").is_file():
87             return directory, "pyproject.toml"
88
89     return directory, "file system root"
90
91
92 def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]:
93     """Find the absolute filepath to a pyproject.toml if it exists"""
94     path_project_root, _ = find_project_root(path_search_start)
95     path_pyproject_toml = path_project_root / "pyproject.toml"
96     if path_pyproject_toml.is_file():
97         return str(path_pyproject_toml)
98
99     try:
100         path_user_pyproject_toml = find_user_pyproject_toml()
101         return (
102             str(path_user_pyproject_toml)
103             if path_user_pyproject_toml.is_file()
104             else None
105         )
106     except (PermissionError, RuntimeError) as e:
107         # We do not have access to the user-level config directory, so ignore it.
108         err(f"Ignoring user configuration directory due to {e!r}")
109         return None
110
111
112 @mypyc_attr(patchable=True)
113 def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
114     """Parse a pyproject toml file, pulling out relevant parts for Black.
115
116     If parsing fails, will raise a tomllib.TOMLDecodeError.
117     """
118     with open(path_config, "rb") as f:
119         pyproject_toml = tomllib.load(f)
120     config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {})
121     config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
122
123     if "target_version" not in config:
124         inferred_target_version = infer_target_version(pyproject_toml)
125         if inferred_target_version is not None:
126             config["target_version"] = [v.name.lower() for v in inferred_target_version]
127
128     return config
129
130
131 def infer_target_version(
132     pyproject_toml: Dict[str, Any]
133 ) -> Optional[List[TargetVersion]]:
134     """Infer Black's target version from the project metadata in pyproject.toml.
135
136     Supports the PyPA standard format (PEP 621):
137     https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#requires-python
138
139     If the target version cannot be inferred, returns None.
140     """
141     project_metadata = pyproject_toml.get("project", {})
142     requires_python = project_metadata.get("requires-python", None)
143     if requires_python is not None:
144         try:
145             return parse_req_python_version(requires_python)
146         except InvalidVersion:
147             pass
148         try:
149             return parse_req_python_specifier(requires_python)
150         except (InvalidSpecifier, InvalidVersion):
151             pass
152
153     return None
154
155
156 def parse_req_python_version(requires_python: str) -> Optional[List[TargetVersion]]:
157     """Parse a version string (i.e. ``"3.7"``) to a list of TargetVersion.
158
159     If parsing fails, will raise a packaging.version.InvalidVersion error.
160     If the parsed version cannot be mapped to a valid TargetVersion, returns None.
161     """
162     version = Version(requires_python)
163     if version.release[0] != 3:
164         return None
165     try:
166         return [TargetVersion(version.release[1])]
167     except (IndexError, ValueError):
168         return None
169
170
171 def parse_req_python_specifier(requires_python: str) -> Optional[List[TargetVersion]]:
172     """Parse a specifier string (i.e. ``">=3.7,<3.10"``) to a list of TargetVersion.
173
174     If parsing fails, will raise a packaging.specifiers.InvalidSpecifier error.
175     If the parsed specifier cannot be mapped to a valid TargetVersion, returns None.
176     """
177     specifier_set = strip_specifier_set(SpecifierSet(requires_python))
178     if not specifier_set:
179         return None
180
181     target_version_map = {f"3.{v.value}": v for v in TargetVersion}
182     compatible_versions: List[str] = list(specifier_set.filter(target_version_map))
183     if compatible_versions:
184         return [target_version_map[v] for v in compatible_versions]
185     return None
186
187
188 def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet:
189     """Strip minor versions for some specifiers in the specifier set.
190
191     For background on version specifiers, see PEP 440:
192     https://peps.python.org/pep-0440/#version-specifiers
193     """
194     specifiers = []
195     for s in specifier_set:
196         if "*" in str(s):
197             specifiers.append(s)
198         elif s.operator in ["~=", "==", ">=", "==="]:
199             version = Version(s.version)
200             stripped = Specifier(f"{s.operator}{version.major}.{version.minor}")
201             specifiers.append(stripped)
202         elif s.operator == ">":
203             version = Version(s.version)
204             if len(version.release) > 2:
205                 s = Specifier(f">={version.major}.{version.minor}")
206             specifiers.append(s)
207         else:
208             specifiers.append(s)
209
210     return SpecifierSet(",".join(str(s) for s in specifiers))
211
212
213 @lru_cache()
214 def find_user_pyproject_toml() -> Path:
215     r"""Return the path to the top-level user configuration for black.
216
217     This looks for ~\.black on Windows and ~/.config/black on Linux and other
218     Unix systems.
219
220     May raise:
221     - RuntimeError: if the current user has no homedir
222     - PermissionError: if the current process cannot access the user's homedir
223     """
224     if sys.platform == "win32":
225         # Windows
226         user_config_path = Path.home() / ".black"
227     else:
228         config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config")
229         user_config_path = Path(config_root).expanduser() / "black"
230     return user_config_path.resolve()
231
232
233 @lru_cache()
234 def get_gitignore(root: Path) -> PathSpec:
235     """Return a PathSpec matching gitignore content if present."""
236     gitignore = root / ".gitignore"
237     lines: List[str] = []
238     if gitignore.is_file():
239         with gitignore.open(encoding="utf-8") as gf:
240             lines = gf.readlines()
241     try:
242         return PathSpec.from_lines("gitwildmatch", lines)
243     except GitWildMatchPatternError as e:
244         err(f"Could not parse {gitignore}: {e}")
245         raise
246
247
248 def normalize_path_maybe_ignore(
249     path: Path,
250     root: Path,
251     report: Optional[Report] = None,
252 ) -> Optional[str]:
253     """Normalize `path`. May return `None` if `path` was ignored.
254
255     `report` is where "path ignored" output goes.
256     """
257     try:
258         abspath = path if path.is_absolute() else Path.cwd() / path
259         normalized_path = abspath.resolve()
260         try:
261             root_relative_path = normalized_path.relative_to(root).as_posix()
262         except ValueError:
263             if report:
264                 report.path_ignored(
265                     path, f"is a symbolic link that points outside {root}"
266                 )
267             return None
268
269     except OSError as e:
270         if report:
271             report.path_ignored(path, f"cannot be read because {e}")
272         return None
273
274     return root_relative_path
275
276
277 def path_is_ignored(
278     path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report
279 ) -> bool:
280     for gitignore_path, pattern in gitignore_dict.items():
281         relative_path = normalize_path_maybe_ignore(path, gitignore_path, report)
282         if relative_path is None:
283             break
284         if pattern.match_file(relative_path):
285             report.path_ignored(path, "matches a .gitignore file content")
286             return True
287     return False
288
289
290 def path_is_excluded(
291     normalized_path: str,
292     pattern: Optional[Pattern[str]],
293 ) -> bool:
294     match = pattern.search(normalized_path) if pattern else None
295     return bool(match and match.group(0))
296
297
298 def gen_python_files(
299     paths: Iterable[Path],
300     root: Path,
301     include: Pattern[str],
302     exclude: Pattern[str],
303     extend_exclude: Optional[Pattern[str]],
304     force_exclude: Optional[Pattern[str]],
305     report: Report,
306     gitignore_dict: Optional[Dict[Path, PathSpec]],
307     *,
308     verbose: bool,
309     quiet: bool,
310 ) -> Iterator[Path]:
311     """Generate all files under `path` whose paths are not excluded by the
312     `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
313     but are included by the `include` regex.
314
315     Symbolic links pointing outside of the `root` directory are ignored.
316
317     `report` is where output about exclusions goes.
318     """
319
320     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
321     for child in paths:
322         normalized_path = normalize_path_maybe_ignore(child, root, report)
323         if normalized_path is None:
324             continue
325
326         # First ignore files matching .gitignore, if passed
327         if gitignore_dict and path_is_ignored(child, gitignore_dict, report):
328             continue
329
330         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
331         normalized_path = "/" + normalized_path
332         if child.is_dir():
333             normalized_path += "/"
334
335         if path_is_excluded(normalized_path, exclude):
336             report.path_ignored(child, "matches the --exclude regular expression")
337             continue
338
339         if path_is_excluded(normalized_path, extend_exclude):
340             report.path_ignored(
341                 child, "matches the --extend-exclude regular expression"
342             )
343             continue
344
345         if path_is_excluded(normalized_path, force_exclude):
346             report.path_ignored(child, "matches the --force-exclude regular expression")
347             continue
348
349         if child.is_dir():
350             # If gitignore is None, gitignore usage is disabled, while a Falsey
351             # gitignore is when the directory doesn't have a .gitignore file.
352             if gitignore_dict is not None:
353                 new_gitignore_dict = {
354                     **gitignore_dict,
355                     root / child: get_gitignore(child),
356                 }
357             else:
358                 new_gitignore_dict = None
359             yield from gen_python_files(
360                 child.iterdir(),
361                 root,
362                 include,
363                 exclude,
364                 extend_exclude,
365                 force_exclude,
366                 report,
367                 new_gitignore_dict,
368                 verbose=verbose,
369                 quiet=quiet,
370             )
371
372         elif child.is_file():
373             if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
374                 verbose=verbose, quiet=quiet
375             ):
376                 continue
377             include_match = include.search(normalized_path) if include else True
378             if include_match:
379                 yield child
380
381
382 def wrap_stream_for_windows(
383     f: io.TextIOWrapper,
384 ) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]:
385     """
386     Wrap stream with colorama's wrap_stream so colors are shown on Windows.
387
388     If `colorama` is unavailable, the original stream is returned unmodified.
389     Otherwise, the `wrap_stream()` function determines whether the stream needs
390     to be wrapped for a Windows environment and will accordingly either return
391     an `AnsiToWin32` wrapper or the original stream.
392     """
393     try:
394         from colorama.initialise import wrap_stream
395     except ImportError:
396         return f
397     else:
398         # Set `strip=False` to avoid needing to modify test_express_diff_with_color.
399         return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True)