X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/e4340f5c3edcfc2b7ee1ebad4521eeb6deb616df..0c77c58722acc7b594bfb3e12aa4b78a51109b27:/black.py?ds=inline diff --git a/black.py b/black.py index f51b10e..d688748 100644 --- a/black.py +++ b/black.py @@ -20,6 +20,7 @@ from typing import ( Callable, Collection, Dict, + Generator, Generic, Iterable, Iterator, @@ -29,7 +30,6 @@ from typing import ( Sequence, Set, Tuple, - Type, TypeVar, Union, cast, @@ -47,7 +47,7 @@ from blib2to3.pgen2 import driver, token from blib2to3.pgen2.parse import ParseError -__version__ = "18.6b2" +__version__ = "18.6b4" DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = ( r"/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/" @@ -90,43 +90,16 @@ class CannotSplit(Exception): """ -class FormatError(Exception): - """Base exception for `# fmt: on` and `# fmt: off` handling. - - It holds the number of bytes of the prefix consumed before the format - control comment appeared. - """ - - def __init__(self, consumed: int) -> None: - super().__init__(consumed) - self.consumed = consumed - - def trim_prefix(self, leaf: Leaf) -> None: - leaf.prefix = leaf.prefix[self.consumed :] - - def leaf_from_consumed(self, leaf: Leaf) -> Leaf: - """Returns a new Leaf from the consumed part of the prefix.""" - unformatted_prefix = leaf.prefix[: self.consumed] - return Leaf(token.NEWLINE, unformatted_prefix) - - -class FormatOn(FormatError): - """Found a comment like `# fmt: on` in the file.""" - - -class FormatOff(FormatError): - """Found a comment like `# fmt: off` in the file.""" - - class WriteBack(Enum): NO = 0 YES = 1 DIFF = 2 + CHECK = 3 @classmethod def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack": if check and not diff: - return cls.NO + return cls.CHECK return cls.DIFF if diff else cls.YES @@ -426,7 +399,9 @@ def reformat_one( mode=mode, ): changed = Changed.YES - if write_back == WriteBack.YES and changed is not Changed.NO: + if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( + write_back is WriteBack.CHECK and changed is Changed.NO + ): write_cache(cache, [src], line_length, mode) report.done(src, changed) except Exception as exc: @@ -456,50 +431,58 @@ async def schedule_formatting( sources, cached = filter_cached(cache, sources) for src in sorted(cached): report.done(src, Changed.CACHED) + if not sources: + return + cancelled = [] - formatted = [] - if sources: - lock = None - if write_back == WriteBack.DIFF: - # For diff output, we need locks to ensure we don't interleave output - # from different processes. - manager = Manager() - lock = manager.Lock() - tasks = { - loop.run_in_executor( - executor, - format_file_in_place, - src, - line_length, - fast, - write_back, - mode, - lock, - ): src - for src in sorted(sources) - } - pending: Iterable[asyncio.Task] = tasks.keys() - try: - loop.add_signal_handler(signal.SIGINT, cancel, pending) - loop.add_signal_handler(signal.SIGTERM, cancel, pending) - except NotImplementedError: - # There are no good alternatives for these on Windows - pass - while pending: - done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) - for task in done: - src = tasks.pop(task) - if task.cancelled(): - cancelled.append(task) - elif task.exception(): - report.failed(src, str(task.exception())) - else: - formatted.append(src) - report.done(src, Changed.YES if task.result() else Changed.NO) + sources_to_cache = [] + lock = None + if write_back == WriteBack.DIFF: + # For diff output, we need locks to ensure we don't interleave output + # from different processes. + manager = Manager() + lock = manager.Lock() + tasks = { + loop.run_in_executor( + executor, + format_file_in_place, + src, + line_length, + fast, + write_back, + mode, + lock, + ): src + for src in sorted(sources) + } + pending: Iterable[asyncio.Task] = tasks.keys() + try: + loop.add_signal_handler(signal.SIGINT, cancel, pending) + loop.add_signal_handler(signal.SIGTERM, cancel, pending) + except NotImplementedError: + # There are no good alternatives for these on Windows. + pass + while pending: + done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + for task in done: + src = tasks.pop(task) + if task.cancelled(): + cancelled.append(task) + elif task.exception(): + report.failed(src, str(task.exception())) + else: + changed = Changed.YES if task.result() else Changed.NO + # If the file was written back or was successfully checked as + # well-formatted, store this information in the cache. + if write_back is WriteBack.YES or ( + write_back is WriteBack.CHECK and changed is Changed.NO + ): + sources_to_cache.append(src) + report.done(src, changed) if cancelled: await asyncio.gather(*cancelled, loop=loop, return_exceptions=True) - if write_back == WriteBack.YES and formatted: - write_cache(cache, formatted, line_length, mode) + if sources_to_cache: + write_cache(cache, sources_to_cache, line_length, mode) def format_file_in_place( @@ -512,7 +495,8 @@ def format_file_in_place( ) -> bool: """Format file under `src` path. Return True if changed. - If `write_back` is True, write reformatted code back to stdout. + If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted + code to the file. `line_length` and `fast` options are passed to :func:`format_file_contents`. """ if src.suffix == ".pyi": @@ -561,7 +545,8 @@ def format_stdin_to_stdout( ) -> bool: """Format file on stdin. Return True if changed. - If `write_back` is True, write reformatted code back to stdout. + If `write_back` is YES, write reformatted code back to stdout. If it is DIFF, + write a diff to stdout. `line_length`, `fast`, `is_pyi`, and `force_py36` arguments are passed to :func:`format_file_contents`. """ @@ -628,10 +613,12 @@ def format_str( is_pyi = bool(mode & FileMode.PYI) py36 = bool(mode & FileMode.PYTHON36) or is_python36(src_node) normalize_strings = not bool(mode & FileMode.NO_STRING_NORMALIZATION) + normalize_fmt_off(src_node) lines = LineGenerator( remove_u_prefix=py36 or "unicode_literals" in future_imports, is_pyi=is_pyi, normalize_strings=normalize_strings, + allow_underscores=py36, ) elt = EmptyLineTracker(is_pyi=is_pyi) empty_line = Line() @@ -758,13 +745,15 @@ class DebugVisitor(Visitor[T]): out(f" {node.value!r}", fg="blue", bold=False) @classmethod - def show(cls, code: str) -> None: + def show(cls, code: Union[str, Leaf, Node]) -> None: """Pretty-print the lib2to3 AST of a given string of `code`. Convenience method for debugging. """ v: DebugVisitor[None] = DebugVisitor() - list(v.visit(lib2to3_parse(code))) + if isinstance(code, str): + code = lib2to3_parse(code) + list(v.visit(code)) KEYWORDS = set(keyword.kwlist) @@ -781,6 +770,7 @@ STATEMENT = { syms.classdef, } STANDALONE_COMMENT = 153 +token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT" LOGIC_OPERATORS = {"and", "or"} COMPARATORS = { token.LESS, @@ -1292,55 +1282,6 @@ class Line: return bool(self.leaves or self.comments) -class UnformattedLines(Line): - """Just like :class:`Line` but stores lines which aren't reformatted.""" - - def append(self, leaf: Leaf, preformatted: bool = True) -> None: - """Just add a new `leaf` to the end of the lines. - - The `preformatted` argument is ignored. - - Keeps track of indentation `depth`, which is useful when the user - says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`. - """ - try: - list(generate_comments(leaf)) - except FormatOn as f_on: - self.leaves.append(f_on.leaf_from_consumed(leaf)) - raise - - self.leaves.append(leaf) - if leaf.type == token.INDENT: - self.depth += 1 - elif leaf.type == token.DEDENT: - self.depth -= 1 - - def __str__(self) -> str: - """Render unformatted lines from leaves which were added with `append()`. - - `depth` is not used for indentation in this case. - """ - if not self: - return "\n" - - res = "" - for leaf in self.leaves: - res += str(leaf) - return res - - def append_comment(self, comment: Leaf) -> bool: - """Not implemented in this class. Raises `NotImplementedError`.""" - raise NotImplementedError("Unformatted lines don't store comments separately.") - - def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: - """Does nothing and returns False.""" - return False - - def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: - """Does nothing and returns False.""" - return False - - @dataclass class EmptyLineTracker: """Provides a stateful method that returns the number of potential extra @@ -1362,9 +1303,6 @@ class EmptyLineTracker: This is for separating `def`, `async def` and `class` with extra empty lines (two on module-level). """ - if isinstance(current_line, UnformattedLines): - return 0, 0 - before, after = self._maybe_empty_lines(current_line) before -= self.previous_after self.previous_after = after @@ -1467,8 +1405,9 @@ class LineGenerator(Visitor[Line]): normalize_strings: bool = True current_line: Line = Factory(Line) remove_u_prefix: bool = False + allow_underscores: bool = False - def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]: + def line(self, indent: int = 0) -> Iterator[Line]: """Generate a line. If the line is empty, only emit if it makes sense. @@ -1477,67 +1416,41 @@ class LineGenerator(Visitor[Line]): If any lines were generated, set up a new current_line. """ if not self.current_line: - if self.current_line.__class__ == type: - self.current_line.depth += indent - else: - self.current_line = type(depth=self.current_line.depth + indent) + self.current_line.depth += indent return # Line is empty, don't emit. Creating a new one unnecessary. complete_line = self.current_line - self.current_line = type(depth=complete_line.depth + indent) + self.current_line = Line(depth=complete_line.depth + indent) yield complete_line - def visit(self, node: LN) -> Iterator[Line]: - """Main method to visit `node` and its children. - - Yields :class:`Line` objects. - """ - if isinstance(self.current_line, UnformattedLines): - # File contained `# fmt: off` - yield from self.visit_unformatted(node) - - else: - yield from super().visit(node) - def visit_default(self, node: LN) -> Iterator[Line]: """Default `visit_*()` implementation. Recurses to children of `node`.""" if isinstance(node, Leaf): any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() - try: - for comment in generate_comments(node): - if any_open_brackets: - # any comment within brackets is subject to splitting - self.current_line.append(comment) - elif comment.type == token.COMMENT: - # regular trailing comment - self.current_line.append(comment) - yield from self.line() - - else: - # regular standalone comment - yield from self.line() - - self.current_line.append(comment) - yield from self.line() - - except FormatOff as f_off: - f_off.trim_prefix(node) - yield from self.line(type=UnformattedLines) - yield from self.visit(node) - - except FormatOn as f_on: - # This only happens here if somebody says "fmt: on" multiple - # times in a row. - f_on.trim_prefix(node) - yield from self.visit_default(node) + for comment in generate_comments(node): + if any_open_brackets: + # any comment within brackets is subject to splitting + self.current_line.append(comment) + elif comment.type == token.COMMENT: + # regular trailing comment + self.current_line.append(comment) + yield from self.line() - else: - normalize_prefix(node, inside_brackets=any_open_brackets) - if self.normalize_strings and node.type == token.STRING: - normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix) - normalize_string_quotes(node) - if node.type not in WHITESPACE: - self.current_line.append(node) + else: + # regular standalone comment + yield from self.line() + + self.current_line.append(comment) + yield from self.line() + + normalize_prefix(node, inside_brackets=any_open_brackets) + if self.normalize_strings and node.type == token.STRING: + normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix) + normalize_string_quotes(node) + if node.type == token.NUMBER: + normalize_numeric_literal(node, self.allow_underscores) + if node.type not in WHITESPACE: + self.current_line.append(node) yield from super().visit_default(node) def visit_INDENT(self, node: Node) -> Iterator[Line]: @@ -1634,23 +1547,10 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(leaf) yield from self.line() - def visit_unformatted(self, node: LN) -> Iterator[Line]: - """Used when file contained a `# fmt: off`.""" - if isinstance(node, Node): - for child in node.children: - yield from self.visit(child) - - else: - try: - self.current_line.append(node) - except FormatOn as f_on: - f_on.trim_prefix(node) - yield from self.line() - yield from self.visit(node) - - if node.type == token.ENDMARKER: - # somebody decided not to put a final `# fmt: on` - yield from self.line() + def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: + if not self.current_line.bracket_tracker.any_open_brackets(): + yield from self.line() + yield from self.visit_default(leaf) def __attrs_post_init__(self) -> None: """You are in a twisty little maze of passages.""" @@ -1940,6 +1840,31 @@ def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]: return node +def container_of(leaf: Leaf) -> LN: + """Return `leaf` or one of its ancestors that is the topmost container of it. + + By "container" we mean a node where `leaf` is the very first child. + """ + same_prefix = leaf.prefix + container: LN = leaf + while container: + parent = container.parent + if parent is None: + break + + if parent.children[0].prefix != same_prefix: + break + + if parent.type == syms.file_input: + break + + if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS: + break + + container = parent + return container + + def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int: """Return the priority of the `leaf` delimiter, given a line break after it. @@ -2070,16 +1995,6 @@ def generate_comments(leaf: LN) -> Iterator[Leaf]: """ for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) - if pc.value in FMT_ON: - raise FormatOn(pc.consumed) - - if pc.value in FMT_OFF: - if pc.type == STANDALONE_COMMENT: - raise FormatOff(pc.consumed) - - prev = preceding_leaf(leaf) - if not prev or prev.type in WHITESPACE: # standalone comment in disguise - raise FormatOff(pc.consumed) @dataclass @@ -2091,7 +2006,7 @@ class ProtoComment: @lru_cache(maxsize=4096) -def list_comments(prefix: str, is_endmarker: bool) -> List[ProtoComment]: +def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: result: List[ProtoComment] = [] if not prefix or "#" not in prefix: return result @@ -2152,7 +2067,7 @@ def split_line( If `py36` is True, splitting may generate syntax that is only compatible with Python 3.6 and later. """ - if isinstance(line, UnformattedLines) or line.is_comment: + if line.is_comment: yield line return @@ -2595,6 +2510,63 @@ def normalize_string_quotes(leaf: Leaf) -> None: leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}" +def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None: + """Normalizes numeric (float, int, and complex) literals. + + All letters used in the representation are normalized to lowercase, long number + literals are split using underscores. + """ + text = leaf.value.lower() + if text.startswith(("0o", "0x", "0b")): + # Leave octal, hex, and binary literals alone. + pass + elif "e" in text: + before, after = text.split("e") + sign = "" + if after.startswith("-"): + after = after[1:] + sign = "-" + elif after.startswith("+"): + after = after[1:] + before = format_float_or_int_string(before, allow_underscores) + after = format_int_string(after, allow_underscores) + text = f"{before}e{sign}{after}" + elif text.endswith(("j", "l")): + number = text[:-1] + suffix = text[-1] + text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}" + else: + text = format_float_or_int_string(text, allow_underscores) + leaf.value = text + + +def format_float_or_int_string(text: str, allow_underscores: bool) -> str: + """Formats a float string like "1.0".""" + if "." not in text: + return format_int_string(text, allow_underscores) + + before, after = text.split(".") + before = format_int_string(before, allow_underscores) if before else "0" + after = format_int_string(after, allow_underscores) if after else "0" + return f"{before}.{after}" + + +def format_int_string(text: str, allow_underscores: bool) -> str: + """Normalizes underscores in a string to e.g. 1_000_000. + + Input must be a string of at least six digits and optional underscores. + """ + if not allow_underscores: + return text + + text = text.replace("_", "") + if len(text) <= 6: + # No underscores for numbers <= 6 digits long. + return text + + return format(int(text), "3_") + + def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None: """Make existing optional parentheses invisible or create new ones. @@ -2643,6 +2615,83 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None: check_lpar = isinstance(child, Leaf) and child.value in parens_after +def normalize_fmt_off(node: Node) -> None: + """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" + try_again = True + while try_again: + try_again = convert_one_fmt_off_pair(node) + + +def convert_one_fmt_off_pair(node: Node) -> bool: + """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. + + Returns True if a pair was converted. + """ + for leaf in node.leaves(): + previous_consumed = 0 + for comment in list_comments(leaf.prefix, is_endmarker=False): + if comment.value in FMT_OFF: + # We only want standalone comments. If there's no previous leaf or + # the previous leaf is indentation, it's a standalone comment in + # disguise. + if comment.type != STANDALONE_COMMENT: + prev = preceding_leaf(leaf) + if prev and prev.type not in WHITESPACE: + continue + + ignored_nodes = list(generate_ignored_nodes(leaf)) + if not ignored_nodes: + continue + + first = ignored_nodes[0] # Can be a container node with the `leaf`. + parent = first.parent + prefix = first.prefix + first.prefix = prefix[comment.consumed :] + hidden_value = ( + comment.value + "\n" + "".join(str(n) for n in ignored_nodes) + ) + if hidden_value.endswith("\n"): + # That happens when one of the `ignored_nodes` ended with a NEWLINE + # leaf (possibly followed by a DEDENT). + hidden_value = hidden_value[:-1] + first_idx = None + for ignored in ignored_nodes: + index = ignored.remove() + if first_idx is None: + first_idx = index + assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" + assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" + parent.insert_child( + first_idx, + Leaf( + STANDALONE_COMMENT, + hidden_value, + prefix=prefix[:previous_consumed] + "\n" * comment.newlines, + ), + ) + return True + + previous_consumed = comment.consumed + + return False + + +def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]: + """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. + + Stops at the end of the block. + """ + container: Optional[LN] = container_of(leaf) + while container is not None and container.type != token.ENDMARKER: + for comment in list_comments(container.prefix, is_endmarker=False): + if comment.value in FMT_ON: + return + + yield container + + container = container.next_sibling + + def maybe_make_parens_invisible_in_atom(node: LN) -> bool: """If it's safe, make the parens in the atom `node` invisible, recursively.""" if ( @@ -2924,7 +2973,23 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf def get_future_imports(node: Node) -> Set[str]: """Return a set of __future__ imports in the file.""" - imports = set() + imports: Set[str] = set() + + def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]: + for child in children: + if isinstance(child, Leaf): + if child.type == token.NAME: + yield child.value + elif child.type == syms.import_as_name: + orig_name = child.children[0] + assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports" + assert orig_name.type == token.NAME, "Invalid syntax parsing imports" + yield orig_name.value + elif child.type == syms.import_as_names: + yield from get_imports_from_children(child.children) + else: + assert False, "Invalid syntax parsing imports" + for child in node.children: if child.type != syms.simple_stmt: break @@ -2943,15 +3008,7 @@ def get_future_imports(node: Node) -> Set[str]: module_name = first_child.children[1] if not isinstance(module_name, Leaf) or module_name.value != "__future__": break - for import_from_child in first_child.children[3:]: - if isinstance(import_from_child, Leaf): - if import_from_child.type == token.NAME: - imports.add(import_from_child.value) - else: - assert import_from_child.type == syms.import_as_names - for leaf in import_from_child.children: - if isinstance(leaf, Leaf) and leaf.type == token.NAME: - imports.add(leaf.value) + imports |= set(get_imports_from_children(first_child.children[3:])) else: break return imports @@ -2967,7 +3024,7 @@ def gen_python_files_in_dir( """Generate all files under `path` whose paths are not excluded by the `exclude` regex, but are included by the `include` regex. - Symbolic links pointing outside of the root directory are ignored. + Symbolic links pointing outside of the `root` directory are ignored. `report` is where output about exclusions goes. """ @@ -2978,8 +3035,7 @@ def gen_python_files_in_dir( except ValueError: if child.is_symlink(): report.path_ignored( - child, - "is a symbolic link that points outside of the root directory", + child, f"is a symbolic link that points outside {root}" ) continue @@ -3499,5 +3555,28 @@ def write_cache( pass +def patch_click() -> None: + """Make Click not crash. + + On certain misconfigured environments, Python 3 selects the ASCII encoding as the + default which restricts paths that it can access during the lifetime of the + application. Click refuses to work in this scenario by raising a RuntimeError. + + In case of Black the likelihood that non-ASCII characters are going to be used in + file paths is minimal since it's Python source code. Moreover, this crash was + spurious on Python 3.7 thanks to PEP 538 and PEP 540. + """ + try: + from click import core + from click import _unicodefun # type: ignore + except ModuleNotFoundError: + return + + for module in (core, _unicodefun): + if hasattr(module, "_verify_python3_env"): + module._verify_python3_env = lambda: None + + if __name__ == "__main__": + patch_click() main()