src/black/files.py

   1 import io
   2 import os
   3 import sys
   4 from functools import lru_cache
   5 from pathlib import Path
   6 from typing import (
   7     TYPE_CHECKING,
   8     Any,
   9     Dict,
  10     Iterable,
  11     Iterator,
  12     List,
  13     Optional,
  14     Pattern,
  15     Sequence,
  16     Tuple,
  17     Union,
  18 )
  19
  20 from mypy_extensions import mypyc_attr
  21 from pathspec import PathSpec
  22 from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
  23
  24 if sys.version_info >= (3, 11):
  25     try:
  26         import tomllib
  27     except ImportError:
  28         # Help users on older alphas
  29         if not TYPE_CHECKING:
  30             import tomli as tomllib
  31 else:
  32     import tomli as tomllib
  33
  34 from black.handle_ipynb_magics import jupyter_dependencies_are_installed
  35 from black.output import err
  36 from black.report import Report
  37
  38 if TYPE_CHECKING:
  39     import colorama  # noqa: F401
  40
  41
  42 @lru_cache()
  43 def find_project_root(
  44     srcs: Sequence[str], stdin_filename: Optional[str] = None
  45 ) -> Tuple[Path, str]:
  46     """Return a directory containing .git, .hg, or pyproject.toml.
  47
  48     That directory will be a common parent of all files and directories
  49     passed in `srcs`.
  50
  51     If no directory in the tree contains a marker that would specify it's the
  52     project root, the root of the file system is returned.
  53
  54     Returns a two-tuple with the first element as the project root path and
  55     the second element as a string describing the method by which the
  56     project root was discovered.
  57     """
  58     if stdin_filename is not None:
  59         srcs = tuple(stdin_filename if s == "-" else s for s in srcs)
  60     if not srcs:
  61         srcs = [str(Path.cwd().resolve())]
  62
  63     path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs]
  64
  65     # A list of lists of parents for each 'src'. 'src' is included as a
  66     # "parent" of itself if it is a directory
  67     src_parents = [
  68         list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs
  69     ]
  70
  71     common_base = max(
  72         set.intersection(*(set(parents) for parents in src_parents)),
  73         key=lambda path: path.parts,
  74     )
  75
  76     for directory in (common_base, *common_base.parents):
  77         if (directory / ".git").exists():
  78             return directory, ".git directory"
  79
  80         if (directory / ".hg").is_dir():
  81             return directory, ".hg directory"
  82
  83         if (directory / "pyproject.toml").is_file():
  84             return directory, "pyproject.toml"
  85
  86     return directory, "file system root"
  87
  88
  89 def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]:
  90     """Find the absolute filepath to a pyproject.toml if it exists"""
  91     path_project_root, _ = find_project_root(path_search_start)
  92     path_pyproject_toml = path_project_root / "pyproject.toml"
  93     if path_pyproject_toml.is_file():
  94         return str(path_pyproject_toml)
  95
  96     try:
  97         path_user_pyproject_toml = find_user_pyproject_toml()
  98         return (
  99             str(path_user_pyproject_toml)
 100             if path_user_pyproject_toml.is_file()
 101             else None
 102         )
 103     except (PermissionError, RuntimeError) as e:
 104         # We do not have access to the user-level config directory, so ignore it.
 105         err(f"Ignoring user configuration directory due to {e!r}")
 106         return None
 107
 108
 109 @mypyc_attr(patchable=True)
 110 def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
 111     """Parse a pyproject toml file, pulling out relevant parts for Black
 112
 113     If parsing fails, will raise a tomllib.TOMLDecodeError
 114     """
 115     with open(path_config, "rb") as f:
 116         pyproject_toml = tomllib.load(f)
 117     config = pyproject_toml.get("tool", {}).get("black", {})
 118     return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
 119
 120
 121 @lru_cache()
 122 def find_user_pyproject_toml() -> Path:
 123     r"""Return the path to the top-level user configuration for black.
 124
 125     This looks for ~\.black on Windows and ~/.config/black on Linux and other
 126     Unix systems.
 127
 128     May raise:
 129     - RuntimeError: if the current user has no homedir
 130     - PermissionError: if the current process cannot access the user's homedir
 131     """
 132     if sys.platform == "win32":
 133         # Windows
 134         user_config_path = Path.home() / ".black"
 135     else:
 136         config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config")
 137         user_config_path = Path(config_root).expanduser() / "black"
 138     return user_config_path.resolve()
 139
 140
 141 @lru_cache()
 142 def get_gitignore(root: Path) -> PathSpec:
 143     """Return a PathSpec matching gitignore content if present."""
 144     gitignore = root / ".gitignore"
 145     lines: List[str] = []
 146     if gitignore.is_file():
 147         with gitignore.open(encoding="utf-8") as gf:
 148             lines = gf.readlines()
 149     try:
 150         return PathSpec.from_lines("gitwildmatch", lines)
 151     except GitWildMatchPatternError as e:
 152         err(f"Could not parse {gitignore}: {e}")
 153         raise
 154
 155
 156 def normalize_path_maybe_ignore(
 157     path: Path,
 158     root: Path,
 159     report: Optional[Report] = None,
 160 ) -> Optional[str]:
 161     """Normalize `path`. May return `None` if `path` was ignored.
 162
 163     `report` is where "path ignored" output goes.
 164     """
 165     try:
 166         abspath = path if path.is_absolute() else Path.cwd() / path
 167         normalized_path = abspath.resolve()
 168         try:
 169             root_relative_path = normalized_path.relative_to(root).as_posix()
 170         except ValueError:
 171             if report:
 172                 report.path_ignored(
 173                     path, f"is a symbolic link that points outside {root}"
 174                 )
 175             return None
 176
 177     except OSError as e:
 178         if report:
 179             report.path_ignored(path, f"cannot be read because {e}")
 180         return None
 181
 182     return root_relative_path
 183
 184
 185 def path_is_ignored(
 186     path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report
 187 ) -> bool:
 188     for gitignore_path, pattern in gitignore_dict.items():
 189         relative_path = normalize_path_maybe_ignore(path, gitignore_path, report)
 190         if relative_path is None:
 191             break
 192         if pattern.match_file(relative_path):
 193             report.path_ignored(path, "matches a .gitignore file content")
 194             return True
 195     return False
 196
 197
 198 def path_is_excluded(
 199     normalized_path: str,
 200     pattern: Optional[Pattern[str]],
 201 ) -> bool:
 202     match = pattern.search(normalized_path) if pattern else None
 203     return bool(match and match.group(0))
 204
 205
 206 def gen_python_files(
 207     paths: Iterable[Path],
 208     root: Path,
 209     include: Pattern[str],
 210     exclude: Pattern[str],
 211     extend_exclude: Optional[Pattern[str]],
 212     force_exclude: Optional[Pattern[str]],
 213     report: Report,
 214     gitignore_dict: Optional[Dict[Path, PathSpec]],
 215     *,
 216     verbose: bool,
 217     quiet: bool,
 218 ) -> Iterator[Path]:
 219     """Generate all files under `path` whose paths are not excluded by the
 220     `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
 221     but are included by the `include` regex.
 222
 223     Symbolic links pointing outside of the `root` directory are ignored.
 224
 225     `report` is where output about exclusions goes.
 226     """
 227
 228     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
 229     for child in paths:
 230         normalized_path = normalize_path_maybe_ignore(child, root, report)
 231         if normalized_path is None:
 232             continue
 233
 234         # First ignore files matching .gitignore, if passed
 235         if gitignore_dict and path_is_ignored(child, gitignore_dict, report):
 236             continue
 237
 238         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
 239         normalized_path = "/" + normalized_path
 240         if child.is_dir():
 241             normalized_path += "/"
 242
 243         if path_is_excluded(normalized_path, exclude):
 244             report.path_ignored(child, "matches the --exclude regular expression")
 245             continue
 246
 247         if path_is_excluded(normalized_path, extend_exclude):
 248             report.path_ignored(
 249                 child, "matches the --extend-exclude regular expression"
 250             )
 251             continue
 252
 253         if path_is_excluded(normalized_path, force_exclude):
 254             report.path_ignored(child, "matches the --force-exclude regular expression")
 255             continue
 256
 257         if child.is_dir():
 258             # If gitignore is None, gitignore usage is disabled, while a Falsey
 259             # gitignore is when the directory doesn't have a .gitignore file.
 260             if gitignore_dict is not None:
 261                 new_gitignore_dict = {
 262                     **gitignore_dict,
 263                     root / child: get_gitignore(child),
 264                 }
 265             else:
 266                 new_gitignore_dict = None
 267             yield from gen_python_files(
 268                 child.iterdir(),
 269                 root,
 270                 include,
 271                 exclude,
 272                 extend_exclude,
 273                 force_exclude,
 274                 report,
 275                 new_gitignore_dict,
 276                 verbose=verbose,
 277                 quiet=quiet,
 278             )
 279
 280         elif child.is_file():
 281             if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed(
 282                 verbose=verbose, quiet=quiet
 283             ):
 284                 continue
 285             include_match = include.search(normalized_path) if include else True
 286             if include_match:
 287                 yield child
 288
 289
 290 def wrap_stream_for_windows(
 291     f: io.TextIOWrapper,
 292 ) -> Union[io.TextIOWrapper, "colorama.AnsiToWin32"]:
 293     """
 294     Wrap stream with colorama's wrap_stream so colors are shown on Windows.
 295
 296     If `colorama` is unavailable, the original stream is returned unmodified.
 297     Otherwise, the `wrap_stream()` function determines whether the stream needs
 298     to be wrapped for a Windows environment and will accordingly either return
 299     an `AnsiToWin32` wrapper or the original stream.
 300     """
 301     try:
 302         from colorama.initialise import wrap_stream
 303     except ImportError:
 304         return f
 305     else:
 306         # Set `strip=False` to avoid needing to modify test_express_diff_with_color.
 307         return wrap_stream(f, convert=None, strip=False, autoreset=False, wrap=True)