]> git.madduck.net Git - etc/vim.git/blob - src/black/files.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Drop support for parsing Python 2 (#3933)
[etc/vim.git] / src / black / files.py
1 import io
2 import os
3 import sys
4 from functools import lru_cache
5 from pathlib import Path
6 from typing import (
7     TYPE_CHECKING,
8     Any,
9     Dict,
10     Iterable,
11     Iterator,
12     List,
13     Optional,
14     Pattern,
15     Sequence,
16     Tuple,
17     Union,
18 )
19
20 from mypy_extensions import mypyc_attr
21 from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
22 from packaging.version import InvalidVersion, Version
23 from pathspec import PathSpec
24 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
25
26 if sys.version_info >= (3, 11):
27     try:
28         import tomllib
29     except ImportError:
30         # Help users on older alphas
31         if not TYPE_CHECKING:
32             import tomli as tomllib
33 else:
34     import tomli as tomllib
35
36 from black.handle_ipynb_magics import jupyter_dependencies_are_installed
37 from black.mode import TargetVersion
38 from black.output import err
39 from black.report import Report
40
41 if TYPE_CHECKING:
42     import colorama  # noqa: F401
43
44
45 @lru_cache
46 def find_project_root(
47     srcs: Sequence[str], stdin_filename: Optional[str] = None
48 ) -> Tuple[Path, str]:
49     """Return a directory containing .git, .hg, or pyproject.toml.
50
51     That directory will be a common parent of all files and directories
52     passed in `srcs`.
53
54     If no directory in the tree contains a marker that would specify it's the
55     project root, the root of the file system is returned.
56
57     Returns a two-tuple with the first element as the project root path and
58     the second element as a string describing the method by which the
59     project root was discovered.
60     """
61     if stdin_filename is not None:
62         srcs = tuple(stdin_filename if s == "-" else s for s in srcs)
63     if not srcs:
64         srcs = [str(Path.cwd().resolve())]
65
66     path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs]
67
68     # A list of lists of parents for each 'src'. 'src' is included as a
69     # "parent" of itself if it is a directory
70     src_parents = [
71         list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs
72     ]
73
74     common_base = max(
75         set.intersection(*(set(parents) for parents in src_parents)),
76         key=lambda path: path.parts,
77     )
78
79     for directory in (common_base, *common_base.parents):
80         if (directory / ".git").exists():
81             return directory, ".git directory"
82
83         if (directory / ".hg").is_dir():
84             return directory, ".hg directory"
85
86         if (directory / "pyproject.toml").is_file():
87             return directory, "pyproject.toml"
88
89     return directory, "file system root"
90
91
92 def find_pyproject_toml(
93     path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None
94 ) -> Optional[str]:
95     """Find the absolute filepath to a pyproject.toml if it exists"""
96     path_project_root, _ = find_project_root(path_search_start, stdin_filename)
97     path_pyproject_toml = path_project_root / "pyproject.toml"
98     if path_pyproject_toml.is_file():
99         return str(path_pyproject_toml)
100
101     try:
102         path_user_pyproject_toml = find_user_pyproject_toml()
103         return (
104             str(path_user_pyproject_toml)
105             if path_user_pyproject_toml.is_file()
106             else None
107         )
108     except (PermissionError, RuntimeError) as e:
109         # We do not have access to the user-level config directory, so ignore it.
110         err(f"Ignoring user configuration directory due to {e!r}")
111         return None
112
113
114 @mypyc_attr(patchable=True)
115 def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
116     """Parse a pyproject toml file, pulling out relevant parts for Black.
117
118     If parsing fails, will raise a tomllib.TOMLDecodeError.
119     """
120     with open(path_config, "rb") as f:
121         pyproject_toml = tomllib.load(f)
122     config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {})
123     config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
124
125     if "target_version" not in config:
126         inferred_target_version = infer_target_version(pyproject_toml)
127         if inferred_target_version is not None:
128             config["target_version"] = [v.name.lower() for v in inferred_target_version]
129
130     return config
131
132
133 def infer_target_version(
134     pyproject_toml: Dict[str, Any]
135 ) -> Optional[List[TargetVersion]]:
136     """Infer Black's target version from the project metadata in pyproject.toml.
137
138     Supports the PyPA standard format (PEP 621):
139     https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#requires-python
140
141     If the target version cannot be inferred, returns None.
142     """
143     project_metadata = pyproject_toml.get("project", {})
144     requires_python = project_metadata.get("requires-python", None)
145     if requires_python is not None:
146         try:
147             return parse_req_python_version(requires_python)
148         except InvalidVersion:
149             pass
150         try:
151             return parse_req_python_specifier(requires_python)
152         except (InvalidSpecifier, InvalidVersion):
153             pass
154
155     return None
156
157
158 def parse_req_python_version(requires_python: str) -> Optional[List[TargetVersion]]:
159     """Parse a version string (i.e. ``"3.7"``) to a list of TargetVersion.
160
161     If parsing fails, will raise a packaging.version.InvalidVersion error.
162     If the parsed version cannot be mapped to a valid TargetVersion, returns None.
163     """
164     version = Version(requires_python)
165     if version.release[0] != 3:
166         return None
167     try:
168         return [TargetVersion(version.release[1])]
169     except (IndexError, ValueError):
170         return None
171
172
173 def parse_req_python_specifier(requires_python: str) -> Optional[List[TargetVersion]]:
174     """Parse a specifier string (i.e. ``">=3.7,<3.10"``) to a list of TargetVersion.
175
176     If parsing fails, will raise a packaging.specifiers.InvalidSpecifier error.
177     If the parsed specifier cannot be mapped to a valid TargetVersion, returns None.
178     """
179     specifier_set = strip_specifier_set(SpecifierSet(requires_python))
180     if not specifier_set:
181         return None
182
183     target_version_map = {f"3.{v.value}": v for v in TargetVersion}
184     compatible_versions: List[str] = list(specifier_set.filter(target_version_map))
185     if compatible_versions:
186         return [target_version_map[v] for v in compatible_versions]
187     return None
188
189
190 def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet:
191     """Strip minor versions for some specifiers in the specifier set.
192
193     For background on version specifiers, see PEP 440:
194     https://peps.python.org/pep-0440/#version-specifiers
195     """
196     specifiers = []
197     for s in specifier_set:
198         if "*" in str(s):
199             specifiers.append(s)
200         elif s.operator in ["~=", "==", ">=", "==="]:
201             version = Version(s.version)
202             stripped = Specifier(f"{s.operator}{version.major}.{version.minor}")
203             specifiers.append(stripped)
204         elif s.operator == ">":
205             version = Version(s.version)
206             if len(version.release) > 2:
207                 s = Specifier(f">={version.major}.{version.minor}")
208             specifiers.append(s)
209         else:
210             specifiers.append(s)
211
212     return SpecifierSet(",".join(str(s) for s in specifiers))
213
214
215 @lru_cache
216 def find_user_pyproject_toml() -> Path:
217     r"""Return the path to the top-level user configuration for black.
218
219     This looks for ~\.black on Windows and ~/.config/black on Linux and other
220     Unix systems.
221
222     May raise:
223     - RuntimeError: if the current user has no homedir
224     - PermissionError: if the current process cannot access the user's homedir
225     """
226     if sys.platform == "win32":
227         # Windows
228         user_config_path = Path.home() / ".black"
229     else:
230         config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config")
231         user_config_path = Path(config_root).expanduser() / "black"
232     return user_config_path.resolve()
233
234
235 @lru_cache
236 def get_gitignore(root: Path) -> PathSpec:
237     """Return a PathSpec matching gitignore content if present."""
238     gitignore = root / ".gitignore"
239     lines: List[str] = []
240     if gitignore.is_file():
241         with gitignore.open(encoding="utf-8") as gf:
242             lines = gf.readlines()
243     try:
244         return PathSpec.from_lines("gitwildmatch", lines)
245     except GitWildMatchPatternError as e:
246         err(f"Could not parse {gitignore}: {e}")
247         raise
248
249
250 def normalize_path_maybe_ignore(
251     path: Path,
252     root: Path,
253     report: Optional[Report] = None,
254 ) -> Optional[str]:
255     """Normalize `path`. May return `None` if `path` was ignored.
256
257     `report` is where "path ignored" output goes.
258     """
259     try:
260         abspath = path if path.is_absolute() else Path.cwd() / path
261         normalized_path = abspath.resolve()
262         try:
263             root_relative_path = normalized_path.relative_to(root).as_posix()
264         except ValueError:
265             if report:
266                 report.path_ignored(
267                     path, f"is a symbolic link that points outside {root}"
268                 )
269             return None
270
271     except OSError as e:
272         if report:
273             report.path_ignored(path, f"cannot be read because {e}")
274         return None
275
276     return root_relative_path
277
278
279 def _path_is_ignored(
280     root_relative_path: str,
281     root: Path,
282     gitignore_dict: Dict[Path, PathSpec],
283     report: Report,
284 ) -> bool:
285     path = root / root_relative_path
286     # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must
287     # ensure that gitignore_dict is ordered from least specific to most specific.
288     for gitignore_path, pattern in gitignore_dict.items():
289         try:
290             relative_path = path.relative_to(gitignore_path).as_posix()
291         except ValueError:
292             break
293         if pattern.match_file(relative_path):
294             report.path_ignored(
295                 path.relative_to(root), "matches a .gitignore file content"
296             )
297             return True
298     return False
299
300
301 def path_is_excluded(
302     normalized_path: str,
303     pattern: Optional[Pattern[str]],
304 ) -> bool:
305     match = pattern.search(normalized_path) if pattern else None
306     return bool(match and match.group(0))
307
308
309 def gen_python_files(
310     paths: Iterable[Path],
311     root: Path,
312     include: Pattern[str],
313     exclude: Pattern[str],
314     extend_exclude: Optional[Pattern[str]],
315     force_exclude: Optional[Pattern[str]],
316     report: Report,
317     gitignore_dict: Optional[Dict[Path, PathSpec]],
318     *,
319     verbose: bool,
320     quiet: bool,
321 ) -> Iterator[Path]:
322     """Generate all files under `path` whose paths are not excluded by the
323     `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
324     but are included by the `include` regex.
325
326     Symbolic links pointing outside of the `root` directory are ignored.
327
328     `report` is where output about exclusions goes.
329     """
330
331     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
332     for child in paths:
333         root_relative_path = child.absolute().relative_to(root).as_posix()
334
335         # First ignore files matching .gitignore, if passed
336         if gitignore_dict and _path_is_ignored(
337             root_relative_path, root, gitignore_dict, report
338         ):
339             continue
340
341         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
342         root_relative_path = "/" + root_relative_path
343         if child.is_dir():
344             root_relative_path += "/"
345
346         if path_is_excluded(root_relative_path, exclude):
347             report.path_ignored(child, "matches the --exclude regular expression")
348             continue
349
350         if path_is_excluded(root_relative_path, extend_exclude):
351             report.path_ignored(
352                 child, "matches the --extend-exclude regular expression"
353             )
354             continue
355
356         if path_is_excluded(root_relative_path, force_exclude):
357             report.path_ignored(child, "matches the --force-exclude regular expression")
358             continue
359
360         normalized_path = normalize_path_maybe_ignore(child, root, report)
361         if normalized_path is None:
362             continue
363
364         if child.is_dir():
365             # If gitignore is None, gitignore usage is disabled, while a Falsey
366             # gitignore is when the directory doesn't have a .gitignore file.
367             if gitignore_dict is not None:
368                 new_gitignore_dict = {
369                     **gitignore_dict,
370                     root / child: get_gitignore(child),
371                 }
372             else:
373                 new_gitignore_dict = None
374             yield from gen_python_files(
375                 child.iterdir(),
376                 root,
377                 include,
378                 exclude,
379                 extend_exclude,
380                 force_exclude,
381                 report,
382                 new_gitignore_dict,
383                 verbose=verbose,
384                 quiet=quiet,
385             )
386
387         elif child.is_file():
388             if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
389                 warn=verbose or not quiet
390             ):
391                 continue
392             include_match = include.search(normalized_path) if include else True
393             if include_match:
394                 yield child
395
396
397 def wrap_stream_for_windows(
398     f: io.TextIOWrapper,
399 ) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]:
400     """
401     Wrap stream with colorama's wrap_stream so colors are shown on Windows.
402
403     If `colorama` is unavailable, the original stream is returned unmodified.
404     Otherwise, the `wrap_stream()` function determines whether the stream needs
405     to be wrapped for a Windows environment and will accordingly either return
406     an `AnsiToWin32` wrapper or the original stream.
407     """
408     try:
409         from colorama.initialise import wrap_stream
410     except ImportError:
411         return f
412     else:
413         # Set `strip=False` to avoid needing to modify test_express_diff_with_color.
414         return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True)