X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/51756a405cd6006ef22e9c12f212905fe0907f80..ee02ebe3e9996345acd4c042f7b8daffb686167b:/black.py diff --git a/black.py b/black.py index ce4a44f..19a023c 100644 --- a/black.py +++ b/black.py @@ -4,6 +4,7 @@ from asyncio.base_events import BaseEventLoop from concurrent.futures import Executor, ProcessPoolExecutor from enum import Enum, Flag from functools import partial, wraps +import io import keyword import logging from multiprocessing import Manager @@ -47,7 +48,7 @@ from blib2to3.pgen2.parse import ParseError __version__ = "18.5b1" DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = ( - r"build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" + r"/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/" ) DEFAULT_INCLUDES = r"\.pyi?$" CACHE_DIR = Path(user_cache_dir("black", version=__version__)) @@ -119,6 +120,13 @@ class WriteBack(Enum): YES = 1 DIFF = 2 + @classmethod + def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack": + if check and not diff: + return cls.NO + + return cls.DIFF if diff else cls.YES + class Changed(Enum): NO = 0 @@ -132,6 +140,19 @@ class FileMode(Flag): PYI = 2 NO_STRING_NORMALIZATION = 4 + @classmethod + def from_configuration( + cls, *, py36: bool, pyi: bool, skip_string_normalization: bool + ) -> "FileMode": + mode = cls.AUTO_DETECT + if py36: + mode |= cls.PYTHON36 + if pyi: + mode |= cls.PYI + if skip_string_normalization: + mode |= cls.NO_STRING_NORMALIZATION + return mode + @click.command() @click.option( @@ -143,55 +164,46 @@ class FileMode(Flag): show_default=True, ) @click.option( - "--check", + "--py36", is_flag=True, help=( - "Don't write the files back, just return the status. Return code 0 " - "means nothing would change. Return code 1 means some files would be " - "reformatted. Return code 123 means there was an internal error." + "Allow using Python 3.6-only syntax on all input files. This will put " + "trailing commas in function signatures and calls also after *args and " + "**kwargs. [default: per-file auto-detection]" ), ) @click.option( - "--diff", - is_flag=True, - help="Don't write the files back, just output a diff for each file on stdout.", -) -@click.option( - "--fast/--safe", + "--pyi", is_flag=True, - help="If --fast given, skip temporary sanity checks. [default: --safe]", + help=( + "Format all input files like typing stubs regardless of file extension " + "(useful when piping source on standard input)." + ), ) @click.option( - "-q", - "--quiet", + "-S", + "--skip-string-normalization", is_flag=True, - help=( - "Don't emit non-error messages to stderr. Errors are still emitted, " - "silence those with 2>/dev/null." - ), + help="Don't normalize string quotes or prefixes.", ) @click.option( - "--pyi", + "--check", is_flag=True, help=( - "Consider all input files typing stubs regardless of file extension " - "(useful when piping source on standard input)." + "Don't write the files back, just return the status. Return code 0 " + "means nothing would change. Return code 1 means some files would be " + "reformatted. Return code 123 means there was an internal error." ), ) @click.option( - "--py36", + "--diff", is_flag=True, - help=( - "Allow using Python 3.6-only syntax on all input files. This will put " - "trailing commas in function signatures and calls also after *args and " - "**kwargs. [default: per-file auto-detection]" - ), + help="Don't write the files back, just output a diff for each file on stdout.", ) @click.option( - "-S", - "--skip-string-normalization", + "--fast/--safe", is_flag=True, - help="Don't normalize string quotes or prefixes.", + help="If --fast given, skip temporary sanity checks. [default: --safe]", ) @click.option( "--include", @@ -199,8 +211,10 @@ class FileMode(Flag): default=DEFAULT_INCLUDES, help=( "A regular expression that matches files and directories that should be " - "included on recursive searches. On Windows, use forward slashes for " - "directories." + "included on recursive searches. An empty value means all files are " + "included regardless of the name. Use forward slashes for directories on " + "all platforms (Windows, too). Exclusions are calculated first, inclusions " + "later." ), show_default=True, ) @@ -210,11 +224,30 @@ class FileMode(Flag): default=DEFAULT_EXCLUDES, help=( "A regular expression that matches files and directories that should be " - "excluded on recursive searches. On Windows, use forward slashes for " - "directories." + "excluded on recursive searches. An empty value means no paths are excluded. " + "Use forward slashes for directories on all platforms (Windows, too). " + "Exclusions are calculated first, inclusions later." ), show_default=True, ) +@click.option( + "-q", + "--quiet", + is_flag=True, + help=( + "Don't emit non-error messages to stderr. Errors are still emitted, " + "silence those with 2>/dev/null." + ), +) +@click.option( + "-v", + "--verbose", + is_flag=True, + help=( + "Also emit messages to stderr about files that were not changed or were " + "ignored due to --exclude=." + ), +) @click.version_option(version=__version__) @click.argument( "src", @@ -234,12 +267,18 @@ def main( py36: bool, skip_string_normalization: bool, quiet: bool, + verbose: bool, include: str, exclude: str, src: List[str], ) -> None: """The uncompromising code formatter.""" - sources: List[Path] = [] + write_back = WriteBack.from_configuration(check=check, diff=diff) + mode = FileMode.from_configuration( + py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization + ) + report = Report(check=check, quiet=quiet, verbose=verbose) + sources: Set[Path] = set() try: include_regex = re.compile(include) except re.error: @@ -250,40 +289,27 @@ def main( except re.error: err(f"Invalid regular expression for exclude given: {exclude!r}") ctx.exit(2) + root = find_project_root(src) for s in src: p = Path(s) if p.is_dir(): - sources.extend(gen_python_files_in_dir(p, include_regex, exclude_regex)) - elif p.is_file(): + sources.update( + gen_python_files_in_dir(p, root, include_regex, exclude_regex, report) + ) + elif p.is_file() or s == "-": # if a file was explicitly given, we don't care about its extension - sources.append(p) - elif s == "-": - sources.append(Path("-")) + sources.add(p) else: err(f"invalid path: {s}") - - if check and not diff: - write_back = WriteBack.NO - elif diff: - write_back = WriteBack.DIFF - else: - write_back = WriteBack.YES - mode = FileMode.AUTO_DETECT - if py36: - mode |= FileMode.PYTHON36 - if pyi: - mode |= FileMode.PYI - if skip_string_normalization: - mode |= FileMode.NO_STRING_NORMALIZATION - report = Report(check=check, quiet=quiet) if len(sources) == 0: - out("No paths given. Nothing to do 😴") + if verbose or not quiet: + out("No paths given. Nothing to do 😴") ctx.exit(0) return elif len(sources) == 1: reformat_one( - src=sources[0], + src=sources.pop(), line_length=line_length, fast=fast, write_back=write_back, @@ -308,9 +334,9 @@ def main( ) finally: shutdown(loop) - if not quiet: - out("All done! ✨ 🍰 ✨") - click.echo(str(report)) + if verbose or not quiet: + out("All done! ✨ 🍰 ✨") + click.echo(str(report)) ctx.exit(report.return_code) @@ -339,8 +365,8 @@ def reformat_one( cache: Cache = {} if write_back != WriteBack.DIFF: cache = read_cache(line_length, mode) - src = src.resolve() - if src in cache and cache[src] == get_cache_info(src): + res_src = src.resolve() + if res_src in cache and cache[res_src] == get_cache_info(res_src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( src, @@ -358,7 +384,7 @@ def reformat_one( async def schedule_formatting( - sources: List[Path], + sources: Set[Path], line_length: int, fast: bool, write_back: WriteBack, @@ -378,7 +404,7 @@ async def schedule_formatting( if write_back != WriteBack.DIFF: cache = read_cache(line_length, mode) sources, cached = filter_cached(cache, sources) - for src in cached: + for src in sorted(cached): report.done(src, Changed.CACHED) cancelled = [] formatted = [] @@ -441,8 +467,9 @@ def format_file_in_place( """ if src.suffix == ".pyi": mode |= FileMode.PYI - with tokenize.open(src) as src_buffer: - src_contents = src_buffer.read() + + with open(src, "rb") as buf: + newline, encoding, src_contents = prepare_input(buf.read()) try: dst_contents = format_file_contents( src_contents, line_length=line_length, fast=fast, mode=mode @@ -451,7 +478,7 @@ def format_file_in_place( return False if write_back == write_back.YES: - with open(src, "w", encoding=src_buffer.encoding) as f: + with open(src, "w", encoding=encoding, newline=newline) as f: f.write(dst_contents) elif write_back == write_back.DIFF: src_name = f"{src} (original)" @@ -460,7 +487,14 @@ def format_file_in_place( if lock: lock.acquire() try: - sys.stdout.write(diff_contents) + f = io.TextIOWrapper( + sys.stdout.buffer, + encoding=encoding, + newline=newline, + write_through=True, + ) + f.write(diff_contents) + f.detach() finally: if lock: lock.release() @@ -479,7 +513,7 @@ def format_stdin_to_stdout( `line_length`, `fast`, `is_pyi`, and `force_py36` arguments are passed to :func:`format_file_contents`. """ - src = sys.stdin.read() + newline, encoding, src = prepare_input(sys.stdin.buffer.read()) dst = src try: dst = format_file_contents(src, line_length=line_length, fast=fast, mode=mode) @@ -490,11 +524,25 @@ def format_stdin_to_stdout( finally: if write_back == WriteBack.YES: - sys.stdout.write(dst) + f = io.TextIOWrapper( + sys.stdout.buffer, + encoding=encoding, + newline=newline, + write_through=True, + ) + f.write(dst) + f.detach() elif write_back == WriteBack.DIFF: src_name = " (original)" dst_name = " (formatted)" - sys.stdout.write(diff(src, dst, src_name, dst_name)) + f = io.TextIOWrapper( + sys.stdout.buffer, + encoding=encoding, + newline=newline, + write_through=True, + ) + f.write(diff(src, dst, src_name, dst_name)) + f.detach() def format_file_contents( @@ -555,6 +603,19 @@ def format_str( return dst_contents +def prepare_input(src: bytes) -> Tuple[str, str, str]: + """Analyze `src` and return a tuple of (newline, encoding, decoded_contents) + + Where `newline` is either CRLF or LF, and `decoded_contents` is decoded with + universal newlines (i.e. only LF). + """ + srcbuf = io.BytesIO(src) + encoding, lines = tokenize.detect_encoding(srcbuf.readline) + newline = "\r\n" if b"\r\n" == lines[0][-2:] else "\n" + srcbuf.seek(0) + return newline, encoding, io.TextIOWrapper(srcbuf, encoding).read() + + GRAMMARS = [ pygram.python_grammar_no_print_statement_no_exec_statement, pygram.python_grammar_no_print_statement, @@ -566,8 +627,7 @@ def lib2to3_parse(src_txt: str) -> Node: """Given a string with source, return the lib2to3 Node.""" grammar = pygram.python_grammar_no_print_statement if src_txt[-1] != "\n": - nl = "\r\n" if "\r\n" in src_txt[:1024] else "\n" - src_txt += nl + src_txt += "\n" for grammar in GRAMMARS: drv = driver.Driver(grammar, pytree.convert) try: @@ -2789,42 +2849,72 @@ def get_future_imports(node: Node) -> Set[str]: def gen_python_files_in_dir( - path: Path, include: Pattern[str], exclude: Pattern[str] + path: Path, + root: Path, + include: Pattern[str], + exclude: Pattern[str], + report: "Report", ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude` regex, but are included by the `include` regex. - """ + `report` is where output about exclusions goes. + """ + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in path.iterdir(): - searchable_path = str(child.as_posix()) - if Path(child.parts[0]).is_dir(): - searchable_path = "/" + searchable_path + normalized_path = "/" + child.resolve().relative_to(root).as_posix() if child.is_dir(): - searchable_path = searchable_path + "/" - exclude_match = exclude.search(searchable_path) - if exclude_match and len(exclude_match.group()) > 0: - continue + normalized_path += "/" + exclude_match = exclude.search(normalized_path) + if exclude_match and exclude_match.group(0): + report.path_ignored(child, f"matches --exclude={exclude.pattern}") + continue - yield from gen_python_files_in_dir(child, include, exclude) + if child.is_dir(): + yield from gen_python_files_in_dir(child, root, include, exclude, report) - else: - include_match = include.search(searchable_path) - exclude_match = exclude.search(searchable_path) - if ( - child.is_file() - and include_match - and len(include_match.group()) > 0 - and (not exclude_match or len(exclude_match.group()) == 0) - ): + elif child.is_file(): + include_match = include.search(normalized_path) + if include_match: yield child +def find_project_root(srcs: List[str]) -> Path: + """Return a directory containing .git, .hg, or pyproject.toml. + + That directory can be one of the directories passed in `srcs` or their + common parent. + + If no directory in the tree contains a marker that would specify it's the + project root, the root of the file system is returned. + """ + if not srcs: + return Path("/").resolve() + + common_base = min(Path(src).resolve() for src in srcs) + if common_base.is_dir(): + # Append a fake file so `parents` below returns `common_base_dir`, too. + common_base /= "fake-file" + for directory in common_base.parents: + if (directory / ".git").is_dir(): + return directory + + if (directory / ".hg").is_dir(): + return directory + + if (directory / "pyproject.toml").is_file(): + return directory + + return directory + + @dataclass class Report: """Provides a reformatting counter. Can be rendered with `str(report)`.""" check: bool = False quiet: bool = False + verbose: bool = False change_count: int = 0 same_count: int = 0 failure_count: int = 0 @@ -2833,11 +2923,11 @@ class Report: """Increment the counter for successful reformatting. Write out a message.""" if changed is Changed.YES: reformatted = "would reformat" if self.check else "reformatted" - if not self.quiet: + if self.verbose or not self.quiet: out(f"{reformatted} {src}") self.change_count += 1 else: - if not self.quiet: + if self.verbose: if changed is Changed.NO: msg = f"{src} already well formatted, good job." else: @@ -2850,6 +2940,10 @@ class Report: err(f"error: cannot format {src}: {message}") self.failure_count += 1 + def path_ignored(self, path: Path, message: str) -> None: + if self.verbose: + out(f"{path} ignored: {message}", bold=False) + @property def return_code(self) -> int: """Return the exit code that the app should use. @@ -3210,26 +3304,24 @@ def get_cache_info(path: Path) -> CacheInfo: return stat.st_mtime, stat.st_size -def filter_cached( - cache: Cache, sources: Iterable[Path] -) -> Tuple[List[Path], List[Path]]: - """Split a list of paths into two. +def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: + """Split an iterable of paths in `sources` into two sets. - The first list contains paths of files that modified on disk or are not in the - cache. The other list contains paths to non-modified files. + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. """ - todo, done = [], [] + todo, done = set(), set() for src in sources: src = src.resolve() if cache.get(src) != get_cache_info(src): - todo.append(src) + todo.add(src) else: - done.append(src) + done.add(src) return todo, done def write_cache( - cache: Cache, sources: List[Path], line_length: int, mode: FileMode + cache: Cache, sources: Iterable[Path], line_length: int, mode: FileMode ) -> None: """Update the cache file.""" cache_file = get_cache_file(line_length, mode)