]> git.madduck.net Git - etc/vim.git/blobdiff - src/black/files.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Fix diff-shades comment missing newlines (#3799)
[etc/vim.git] / src / black / files.py
index 213bbf82a58e79fc45e4cf67d1773f05953034b7..368e4170d4722d64ec9f8d96f1b1364467ccb15b 100644 (file)
@@ -1,9 +1,10 @@
-from functools import lru_cache
 import io
 import os
-from pathlib import Path
 import sys
+from functools import lru_cache
+from pathlib import Path
 from typing import (
+    TYPE_CHECKING,
     Any,
     Dict,
     Iterable,
@@ -14,12 +15,26 @@ from typing import (
     Sequence,
     Tuple,
     Union,
-    TYPE_CHECKING,
 )
 
+from mypy_extensions import mypyc_attr
+from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
+from packaging.version import InvalidVersion, Version
 from pathspec import PathSpec
-import tomli
+from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
 
+if sys.version_info >= (3, 11):
+    try:
+        import tomllib
+    except ImportError:
+        # Help users on older alphas
+        if not TYPE_CHECKING:
+            import tomli as tomllib
+else:
+    import tomli as tomllib
+
+from black.handle_ipynb_magics import jupyter_dependencies_are_installed
+from black.mode import TargetVersion
 from black.output import err
 from black.report import Report
 
@@ -27,8 +42,10 @@ if TYPE_CHECKING:
     import colorama  # noqa: F401
 
 
-@lru_cache()
-def find_project_root(srcs: Sequence[str]) -> Path:
+@lru_cache
+def find_project_root(
+    srcs: Sequence[str], stdin_filename: Optional[str] = None
+) -> Tuple[Path, str]:
     """Return a directory containing .git, .hg, or pyproject.toml.
 
     That directory will be a common parent of all files and directories
@@ -36,7 +53,13 @@ def find_project_root(srcs: Sequence[str]) -> Path:
 
     If no directory in the tree contains a marker that would specify it's the
     project root, the root of the file system is returned.
+
+    Returns a two-tuple with the first element as the project root path and
+    the second element as a string describing the method by which the
+    project root was discovered.
     """
+    if stdin_filename is not None:
+        srcs = tuple(stdin_filename if s == "-" else s for s in srcs)
     if not srcs:
         srcs = [str(Path.cwd().resolve())]
 
@@ -55,20 +78,22 @@ def find_project_root(srcs: Sequence[str]) -> Path:
 
     for directory in (common_base, *common_base.parents):
         if (directory / ".git").exists():
-            return directory
+            return directory, ".git directory"
 
         if (directory / ".hg").is_dir():
-            return directory
+            return directory, ".hg directory"
 
         if (directory / "pyproject.toml").is_file():
-            return directory
+            return directory, "pyproject.toml"
 
-    return directory
+    return directory, "file system root"
 
 
-def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]:
+def find_pyproject_toml(
+    path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None
+) -> Optional[str]:
     """Find the absolute filepath to a pyproject.toml if it exists"""
-    path_project_root = find_project_root(path_search_start)
+    path_project_root, _ = find_project_root(path_search_start, stdin_filename)
     path_pyproject_toml = path_project_root / "pyproject.toml"
     if path_pyproject_toml.is_file():
         return str(path_pyproject_toml)
@@ -80,29 +105,123 @@ def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]:
             if path_user_pyproject_toml.is_file()
             else None
         )
-    except PermissionError as e:
+    except (PermissionError, RuntimeError) as e:
         # We do not have access to the user-level config directory, so ignore it.
         err(f"Ignoring user configuration directory due to {e!r}")
         return None
 
 
+@mypyc_attr(patchable=True)
 def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
-    """Parse a pyproject toml file, pulling out relevant parts for Black
+    """Parse a pyproject toml file, pulling out relevant parts for Black.
+
+    If parsing fails, will raise a tomllib.TOMLDecodeError.
+    """
+    with open(path_config, "rb") as f:
+        pyproject_toml = tomllib.load(f)
+    config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {})
+    config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
+
+    if "target_version" not in config:
+        inferred_target_version = infer_target_version(pyproject_toml)
+        if inferred_target_version is not None:
+            config["target_version"] = [v.name.lower() for v in inferred_target_version]
+
+    return config
+
+
+def infer_target_version(
+    pyproject_toml: Dict[str, Any]
+) -> Optional[List[TargetVersion]]:
+    """Infer Black's target version from the project metadata in pyproject.toml.
+
+    Supports the PyPA standard format (PEP 621):
+    https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#requires-python
 
-    If parsing fails, will raise a tomli.TOMLDecodeError
+    If the target version cannot be inferred, returns None.
     """
-    with open(path_config, encoding="utf8") as f:
-        pyproject_toml = tomli.load(f)  # type: ignore  # due to deprecated API usage
-    config = pyproject_toml.get("tool", {}).get("black", {})
-    return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
+    project_metadata = pyproject_toml.get("project", {})
+    requires_python = project_metadata.get("requires-python", None)
+    if requires_python is not None:
+        try:
+            return parse_req_python_version(requires_python)
+        except InvalidVersion:
+            pass
+        try:
+            return parse_req_python_specifier(requires_python)
+        except (InvalidSpecifier, InvalidVersion):
+            pass
+
+    return None
+
+
+def parse_req_python_version(requires_python: str) -> Optional[List[TargetVersion]]:
+    """Parse a version string (i.e. ``"3.7"``) to a list of TargetVersion.
+
+    If parsing fails, will raise a packaging.version.InvalidVersion error.
+    If the parsed version cannot be mapped to a valid TargetVersion, returns None.
+    """
+    version = Version(requires_python)
+    if version.release[0] != 3:
+        return None
+    try:
+        return [TargetVersion(version.release[1])]
+    except (IndexError, ValueError):
+        return None
+
 
+def parse_req_python_specifier(requires_python: str) -> Optional[List[TargetVersion]]:
+    """Parse a specifier string (i.e. ``">=3.7,<3.10"``) to a list of TargetVersion.
 
-@lru_cache()
+    If parsing fails, will raise a packaging.specifiers.InvalidSpecifier error.
+    If the parsed specifier cannot be mapped to a valid TargetVersion, returns None.
+    """
+    specifier_set = strip_specifier_set(SpecifierSet(requires_python))
+    if not specifier_set:
+        return None
+
+    target_version_map = {f"3.{v.value}": v for v in TargetVersion}
+    compatible_versions: List[str] = list(specifier_set.filter(target_version_map))
+    if compatible_versions:
+        return [target_version_map[v] for v in compatible_versions]
+    return None
+
+
+def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet:
+    """Strip minor versions for some specifiers in the specifier set.
+
+    For background on version specifiers, see PEP 440:
+    https://peps.python.org/pep-0440/#version-specifiers
+    """
+    specifiers = []
+    for s in specifier_set:
+        if "*" in str(s):
+            specifiers.append(s)
+        elif s.operator in ["~=", "==", ">=", "==="]:
+            version = Version(s.version)
+            stripped = Specifier(f"{s.operator}{version.major}.{version.minor}")
+            specifiers.append(stripped)
+        elif s.operator == ">":
+            version = Version(s.version)
+            if len(version.release) > 2:
+                s = Specifier(f">={version.major}.{version.minor}")
+            specifiers.append(s)
+        else:
+            specifiers.append(s)
+
+    return SpecifierSet(",".join(str(s) for s in specifiers))
+
+
+@lru_cache
 def find_user_pyproject_toml() -> Path:
     r"""Return the path to the top-level user configuration for black.
 
     This looks for ~\.black on Windows and ~/.config/black on Linux and other
     Unix systems.
+
+    May raise:
+    - RuntimeError: if the current user has no homedir
+    - PermissionError: if the current process cannot access the user's homedir
     """
     if sys.platform == "win32":
         # Windows
@@ -113,7 +232,7 @@ def find_user_pyproject_toml() -> Path:
     return user_config_path.resolve()
 
 
-@lru_cache()
+@lru_cache
 def get_gitignore(root: Path) -> PathSpec:
     """Return a PathSpec matching gitignore content if present."""
     gitignore = root / ".gitignore"
@@ -121,11 +240,17 @@ def get_gitignore(root: Path) -> PathSpec:
     if gitignore.is_file():
         with gitignore.open(encoding="utf-8") as gf:
             lines = gf.readlines()
-    return PathSpec.from_lines("gitwildmatch", lines)
+    try:
+        return PathSpec.from_lines("gitwildmatch", lines)
+    except GitWildMatchPatternError as e:
+        err(f"Could not parse {gitignore}: {e}")
+        raise
 
 
 def normalize_path_maybe_ignore(
-    path: Path, root: Path, report: Report
+    path: Path,
+    root: Path,
+    report: Optional[Report] = None,
 ) -> Optional[str]:
     """Normalize `path`. May return `None` if `path` was ignored.
 
@@ -133,19 +258,44 @@ def normalize_path_maybe_ignore(
     """
     try:
         abspath = path if path.is_absolute() else Path.cwd() / path
-        normalized_path = abspath.resolve().relative_to(root).as_posix()
+        normalized_path = abspath.resolve()
+        try:
+            root_relative_path = normalized_path.relative_to(root).as_posix()
+        except ValueError:
+            if report:
+                report.path_ignored(
+                    path, f"is a symbolic link that points outside {root}"
+                )
+            return None
+
     except OSError as e:
-        report.path_ignored(path, f"cannot be read because {e}")
+        if report:
+            report.path_ignored(path, f"cannot be read because {e}")
         return None
 
-    except ValueError:
-        if path.is_symlink():
-            report.path_ignored(path, f"is a symbolic link that points outside {root}")
-            return None
+    return root_relative_path
 
-        raise
 
-    return normalized_path
+def _path_is_ignored(
+    root_relative_path: str,
+    root: Path,
+    gitignore_dict: Dict[Path, PathSpec],
+    report: Report,
+) -> bool:
+    path = root / root_relative_path
+    # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must
+    # ensure that gitignore_dict is ordered from least specific to most specific.
+    for gitignore_path, pattern in gitignore_dict.items():
+        try:
+            relative_path = path.relative_to(gitignore_path).as_posix()
+        except ValueError:
+            break
+        if pattern.match_file(relative_path):
+            report.path_ignored(
+                path.relative_to(root), "matches a .gitignore file content"
+            )
+            return True
+    return False
 
 
 def path_is_excluded(
@@ -164,7 +314,10 @@ def gen_python_files(
     extend_exclude: Optional[Pattern[str]],
     force_exclude: Optional[Pattern[str]],
     report: Report,
-    gitignore: Optional[PathSpec],
+    gitignore_dict: Optional[Dict[Path, PathSpec]],
+    *,
+    verbose: bool,
+    quiet: bool,
 ) -> Iterator[Path]:
     """Generate all files under `path` whose paths are not excluded by the
     `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
@@ -174,6 +327,7 @@ def gen_python_files(
 
     `report` is where output about exclusions goes.
     """
+
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     for child in paths:
         normalized_path = normalize_path_maybe_ignore(child, root, report)
@@ -181,8 +335,9 @@ def gen_python_files(
             continue
 
         # First ignore files matching .gitignore, if passed
-        if gitignore is not None and gitignore.match_file(normalized_path):
-            report.path_ignored(child, "matches the .gitignore file content")
+        if gitignore_dict and _path_is_ignored(
+            normalized_path, root, gitignore_dict, report
+        ):
             continue
 
         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
@@ -207,6 +362,13 @@ def gen_python_files(
         if child.is_dir():
             # If gitignore is None, gitignore usage is disabled, while a Falsey
             # gitignore is when the directory doesn't have a .gitignore file.
+            if gitignore_dict is not None:
+                new_gitignore_dict = {
+                    **gitignore_dict,
+                    root / child: get_gitignore(child),
+                }
+            else:
+                new_gitignore_dict = None
             yield from gen_python_files(
                 child.iterdir(),
                 root,
@@ -215,10 +377,16 @@ def gen_python_files(
                 extend_exclude,
                 force_exclude,
                 report,
-                gitignore + get_gitignore(child) if gitignore is not None else None,
+                new_gitignore_dict,
+                verbose=verbose,
+                quiet=quiet,
             )
 
         elif child.is_file():
+            if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
+                warn=verbose or not quiet
+            ):
+                continue
             include_match = include.search(normalized_path) if include else True
             if include_match:
                 yield child