X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/8e7848c63efe36f09e4651bece8c0efc34a1c3e1..dcd77136f8f62b35007ca11407cd04d7b62daed2:/black.py diff --git a/black.py b/black.py index 587d9b3..e795fa3 100644 --- a/black.py +++ b/black.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import asyncio +import pickle from asyncio.base_events import BaseEventLoop from concurrent.futures import Executor, ProcessPoolExecutor from enum import Enum @@ -24,6 +25,7 @@ from typing import ( Iterator, List, Optional, + Pattern, Set, Tuple, Type, @@ -31,6 +33,7 @@ from typing import ( Union, ) +from appdirs import user_cache_dir from attr import dataclass, Factory import click @@ -40,7 +43,7 @@ from blib2to3 import pygram, pytree from blib2to3.pgen2 import driver, token from blib2to3.pgen2.parse import ParseError -__version__ = "18.4a1" +__version__ = "18.4a2" DEFAULT_LINE_LENGTH = 88 # types syms = pygram.python_symbols @@ -53,6 +56,10 @@ Priority = int Index = int LN = Union[Leaf, Node] SplitFunc = Callable[["Line", bool], Iterator["Line"]] +Timestamp = float +FileSize = int +CacheInfo = Tuple[Timestamp, FileSize] +Cache = Dict[Path, CacheInfo] out = partial(click.secho, bold=True, err=True) err = partial(click.secho, fg="red", err=True) @@ -103,6 +110,12 @@ class WriteBack(Enum): DIFF = 2 +class Changed(Enum): + NO = 0 + CACHED = 1 + YES = 2 + + @click.command() @click.option( "-l", @@ -184,35 +197,70 @@ def main( write_back = WriteBack.YES if len(sources) == 0: ctx.exit(0) + return + elif len(sources) == 1: - p = sources[0] - report = Report(check=check, quiet=quiet) - try: - if not p.is_file() and str(p) == "-": - changed = format_stdin_to_stdout( - line_length=line_length, fast=fast, write_back=write_back - ) - else: - changed = format_file_in_place( - p, line_length=line_length, fast=fast, write_back=write_back - ) - report.done(p, changed) - except Exception as exc: - report.failed(p, str(exc)) - ctx.exit(report.return_code) + return_code = run_single_file_mode( + line_length, check, fast, quiet, write_back, sources[0] + ) else: - loop = asyncio.get_event_loop() - executor = ProcessPoolExecutor(max_workers=os.cpu_count()) - return_code = 1 - try: - return_code = loop.run_until_complete( - schedule_formatting( - sources, line_length, write_back, fast, quiet, loop, executor + return_code = run_multi_file_mode(line_length, fast, quiet, write_back, sources) + ctx.exit(return_code) + + +def run_single_file_mode( + line_length: int, + check: bool, + fast: bool, + quiet: bool, + write_back: WriteBack, + src: Path, +) -> int: + report = Report(check=check, quiet=quiet) + try: + if not src.is_file() and str(src) == "-": + changed = format_stdin_to_stdout( + line_length=line_length, fast=fast, write_back=write_back + ) + else: + changed = Changed.NO + cache: Cache = {} + if write_back != WriteBack.DIFF: + cache = read_cache() + src = src.resolve() + if src in cache and cache[src] == get_cache_info(src): + changed = Changed.CACHED + if changed is not Changed.CACHED: + changed = format_file_in_place( + src, line_length=line_length, fast=fast, write_back=write_back ) + if write_back != WriteBack.DIFF and changed is not Changed.NO: + write_cache(cache, [src]) + report.done(src, changed) + except Exception as exc: + report.failed(src, str(exc)) + return report.return_code + + +def run_multi_file_mode( + line_length: int, + fast: bool, + quiet: bool, + write_back: WriteBack, + sources: List[Path], +) -> int: + loop = asyncio.get_event_loop() + executor = ProcessPoolExecutor(max_workers=os.cpu_count()) + return_code = 1 + try: + return_code = loop.run_until_complete( + schedule_formatting( + sources, line_length, write_back, fast, quiet, loop, executor ) - finally: - shutdown(loop) - ctx.exit(return_code) + ) + finally: + shutdown(loop) + return return_code async def schedule_formatting( @@ -231,41 +279,55 @@ async def schedule_formatting( `line_length`, `write_back`, and `fast` options are passed to :func:`format_file_in_place`. """ - lock = None - if write_back == WriteBack.DIFF: - # For diff output, we need locks to ensure we don't interleave output - # from different processes. - manager = Manager() - lock = manager.Lock() - tasks = { - src: loop.run_in_executor( - executor, format_file_in_place, src, line_length, fast, write_back, lock - ) - for src in sources - } - _task_values = list(tasks.values()) - loop.add_signal_handler(signal.SIGINT, cancel, _task_values) - loop.add_signal_handler(signal.SIGTERM, cancel, _task_values) - await asyncio.wait(tasks.values()) - cancelled = [] report = Report(check=write_back is WriteBack.NO, quiet=quiet) - for src, task in tasks.items(): - if not task.done(): - report.failed(src, "timed out, cancelling") - task.cancel() - cancelled.append(task) - elif task.cancelled(): - cancelled.append(task) - elif task.exception(): - report.failed(src, str(task.exception())) - else: - report.done(src, task.result()) + cache: Cache = {} + if write_back != WriteBack.DIFF: + cache = read_cache() + sources, cached = filter_cached(cache, sources) + for src in cached: + report.done(src, Changed.CACHED) + cancelled = [] + formatted = [] + if sources: + lock = None + if write_back == WriteBack.DIFF: + # For diff output, we need locks to ensure we don't interleave output + # from different processes. + manager = Manager() + lock = manager.Lock() + tasks = { + src: loop.run_in_executor( + executor, format_file_in_place, src, line_length, fast, write_back, lock + ) + for src in sources + } + _task_values = list(tasks.values()) + loop.add_signal_handler(signal.SIGINT, cancel, _task_values) + loop.add_signal_handler(signal.SIGTERM, cancel, _task_values) + await asyncio.wait(_task_values) + for src, task in tasks.items(): + if not task.done(): + report.failed(src, "timed out, cancelling") + task.cancel() + cancelled.append(task) + elif task.cancelled(): + cancelled.append(task) + elif task.exception(): + report.failed(src, str(task.exception())) + else: + formatted.append(src) + report.done(src, task.result()) + if cancelled: await asyncio.gather(*cancelled, loop=loop, return_exceptions=True) elif not quiet: out("All done! ✨ 🍰 ✨") if not quiet: click.echo(str(report)) + + if write_back != WriteBack.DIFF and formatted: + write_cache(cache, formatted) + return report.return_code @@ -275,12 +337,13 @@ def format_file_in_place( fast: bool, write_back: WriteBack = WriteBack.NO, lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy -) -> bool: +) -> Changed: """Format file under `src` path. Return True if changed. If `write_back` is True, write reformatted code back to stdout. `line_length` and `fast` options are passed to :func:`format_file_contents`. """ + with tokenize.open(src) as src_buffer: src_contents = src_buffer.read() try: @@ -288,7 +351,7 @@ def format_file_in_place( src_contents, line_length=line_length, fast=fast ) except NothingChanged: - return False + return Changed.NO if write_back == write_back.YES: with open(src, "w", encoding=src_buffer.encoding) as f: @@ -304,12 +367,12 @@ def format_file_in_place( finally: if lock: lock.release() - return True + return Changed.YES def format_stdin_to_stdout( line_length: int, fast: bool, write_back: WriteBack = WriteBack.NO -) -> bool: +) -> Changed: """Format file on stdin. Return True if changed. If `write_back` is True, write reformatted code back to stdout. @@ -319,10 +382,10 @@ def format_stdin_to_stdout( dst = src try: dst = format_file_contents(src, line_length=line_length, fast=fast) - return True + return Changed.YES except NothingChanged: - return False + return Changed.NO finally: if write_back == WriteBack.YES: @@ -521,7 +584,20 @@ MATH_OPERATORS = { token.DOUBLESTAR, token.DOUBLESLASH, } -VARARGS = {token.STAR, token.DOUBLESTAR} +STARS = {token.STAR, token.DOUBLESTAR} +VARARGS_PARENTS = { + syms.arglist, + syms.argument, # double star in arglist + syms.trailer, # single argument to call + syms.typedargslist, + syms.varargslist, # lambdas +} +UNPACKING_PARENTS = { + syms.atom, # single element of a list or set literal + syms.dictsetmaker, + syms.listmaker, + syms.testlist_gexp, +} COMPREHENSION_PRIORITY = 20 COMMA_PRIORITY = 10 LOGIC_PRIORITY = 5 @@ -1254,18 +1330,8 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 # that, too. return prevp.prefix - elif prevp.type == token.DOUBLESTAR: - if ( - prevp.parent - and prevp.parent.type in { - syms.arglist, - syms.argument, - syms.dictsetmaker, - syms.parameters, - syms.typedargslist, - syms.varargslist, - } - ): + elif prevp.type in STARS: + if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS): return NO elif prevp.type == token.COLON: @@ -1274,7 +1340,7 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 elif ( prevp.parent - and prevp.parent.type in {syms.factor, syms.star_expr} + and prevp.parent.type == syms.factor and prevp.type in MATH_OPERATORS ): return NO @@ -1360,7 +1426,7 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 if not prevp or prevp.type == token.LPAR: return NO - elif prev.type == token.EQUAL or prev.type == token.DOUBLESTAR: + elif prev.type in {token.EQUAL} | STARS: return NO elif p.type == syms.decorator: @@ -1495,11 +1561,7 @@ def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int: Higher numbers are higher priority. """ - if ( - leaf.type in VARARGS - and leaf.parent - and leaf.parent.type in {syms.argument, syms.typedargslist, syms.dictsetmaker} - ): + if is_vararg(leaf, within=VARARGS_PARENTS | UNPACKING_PARENTS): # * and ** might also be MATH_OPERATORS but in this case they are not. # Don't treat them as a delimiter. return 0 @@ -1877,8 +1939,7 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: lowest_depth = min(lowest_depth, leaf.bracket_depth) if ( leaf.bracket_depth == lowest_depth - and leaf.type == token.STAR - or leaf.type == token.DOUBLESTAR + and is_vararg(leaf, within=VARARGS_PARENTS) ): trailing_comma_safe = trailing_comma_safe and py36 leaf_priority = delimiters.get(id(leaf)) @@ -1984,9 +2045,10 @@ def normalize_string_quotes(leaf: Leaf) -> None: return # There's an internal error prefix = leaf.value[:first_quote_pos] - body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)] unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}") - escaped_orig_quote = re.compile(rf"\\(\\\\)*{orig_quote}") + escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}") + escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}") + body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)] if "r" in prefix.casefold(): if unescaped_new_quote.search(body): # There's at least one unescaped new_quote in this raw string @@ -1996,11 +2058,14 @@ def normalize_string_quotes(leaf: Leaf) -> None: # Do not introduce or remove backslashes in raw strings new_body = body else: - new_body = escaped_orig_quote.sub(rf"\1{orig_quote}", body) - new_body = unescaped_new_quote.sub(rf"\1\\{new_quote}", new_body) - # Add escapes again for consecutive occurences of new_quote (sub - # doesn't match overlapping substrings). - new_body = unescaped_new_quote.sub(rf"\1\\{new_quote}", new_body) + # remove unnecessary quotes + new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body) + if body != new_body: + # Consider the string without unnecessary quotes as the original + body = new_body + leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}" + new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body) + new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body) if new_quote == '"""' and new_body[-1] == '"': # edge case: new_body = new_body[:-1] + '\\"' @@ -2085,6 +2150,29 @@ def is_one_tuple(node: LN) -> bool: ) +def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool: + """Return True if `leaf` is a star or double star in a vararg or kwarg. + + If `within` includes VARARGS_PARENTS, this applies to function signatures. + If `within` includes COLLECTION_LIBERALS_PARENTS, it applies to right + hand-side extended iterable unpacking (PEP 3132) and additional unpacking + generalizations (PEP 448). + """ + if leaf.type not in STARS or not leaf.parent: + return False + + p = leaf.parent + if p.type == syms.star_expr: + # Star expressions are also used as assignment targets in extended + # iterable unpacking (PEP 3132). See what its parent is instead. + if not p.parent: + return False + + p = p.parent + + return p.type in within + + def max_delimiter_priority_in_atom(node: LN) -> int: if node.type != syms.atom: return 0 @@ -2139,7 +2227,7 @@ def is_python36(node: Node) -> bool: and n.children[-1].type == token.COMMA ): for ch in n.children: - if ch.type == token.STAR or ch.type == token.DOUBLESTAR: + if ch.type in STARS: return True return False @@ -2175,16 +2263,20 @@ class Report: same_count: int = 0 failure_count: int = 0 - def done(self, src: Path, changed: bool) -> None: + def done(self, src: Path, changed: Changed) -> None: """Increment the counter for successful reformatting. Write out a message.""" - if changed: + if changed is Changed.YES: reformatted = "would reformat" if self.check else "reformatted" if not self.quiet: out(f"{reformatted} {src}") self.change_count += 1 else: if not self.quiet: - out(f"{src} already well formatted, good job.", bold=False) + if changed is Changed.NO: + msg = f"{src} already well formatted, good job." + else: + msg = f"{src} wasn't modified on disk since last run." + out(msg, bold=False) self.same_count += 1 def failed(self, src: Path, message: str) -> None: @@ -2374,5 +2466,71 @@ def shutdown(loop: BaseEventLoop) -> None: loop.close() +def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str: + """Replace `regex` with `replacement` twice on `original`. + + This is used by string normalization to perform replaces on + overlapping matches. + """ + return regex.sub(replacement, regex.sub(replacement, original)) + + +CACHE_DIR = Path(user_cache_dir("black", version=__version__)) +CACHE_FILE = CACHE_DIR / "cache.pickle" + + +def read_cache() -> Cache: + """Read the cache if it exists and is well formed. + + If it is not well formed, the call to write_cache later should resolve the issue. + """ + if not CACHE_FILE.exists(): + return {} + + with CACHE_FILE.open("rb") as fobj: + try: + cache: Cache = pickle.load(fobj) + except pickle.UnpicklingError: + return {} + + return cache + + +def get_cache_info(path: Path) -> CacheInfo: + """Return the information used to check if a file is already formatted or not.""" + stat = path.stat() + return stat.st_mtime, stat.st_size + + +def filter_cached( + cache: Cache, sources: Iterable[Path] +) -> Tuple[List[Path], List[Path]]: + """Split a list of paths into two. + + The first list contains paths of files that modified on disk or are not in the + cache. The other list contains paths to non-modified files. + """ + todo, done = [], [] + for src in sources: + src = src.resolve() + if cache.get(src) != get_cache_info(src): + todo.append(src) + else: + done.append(src) + return todo, done + + +def write_cache(cache: Cache, sources: List[Path]) -> None: + """Update the cache file.""" + try: + if not CACHE_DIR.exists(): + CACHE_DIR.mkdir(parents=True) + new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}} + with CACHE_FILE.open("wb") as fobj: + pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL) + except OSError: + pass + + if __name__ == "__main__": main()