X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/08f1cdd00b4876b2a0545d46981924d5873a3289..9d749280bb1051991d391e0ee70174a613da16fc:/black.py diff --git a/black.py b/black.py index 6f7496e..c4b9132 100644 --- a/black.py +++ b/black.py @@ -5,15 +5,16 @@ from datetime import datetime from enum import Enum, Flag from functools import lru_cache, partial, wraps import io -import keyword +import itertools import logging -from multiprocessing import Manager +from multiprocessing import Manager, freeze_support import os from pathlib import Path import pickle import re import signal import sys +import tempfile import tokenize from typing import ( Any, @@ -47,10 +48,10 @@ from blib2to3.pgen2 import driver, token from blib2to3.pgen2.parse import ParseError -__version__ = "18.6b4" +__version__ = "18.9b0" DEFAULT_LINE_LENGTH = 88 DEFAULT_EXCLUDES = ( - r"/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/" + r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)/" ) DEFAULT_INCLUDES = r"\.pyi?$" CACHE_DIR = Path(user_cache_dir("black", version=__version__)) @@ -79,15 +80,15 @@ syms = pygram.python_symbols class NothingChanged(UserWarning): - """Raised by :func:`format_file` when reformatted code is the same as source.""" + """Raised when reformatted code is the same as source.""" class CannotSplit(Exception): - """A readable split that fits the allotted line length is impossible. + """A readable split that fits the allotted line length is impossible.""" - Raised by :func:`left_hand_split`, :func:`right_hand_split`, and - :func:`delimiter_split`. - """ + +class InvalidInput(ValueError): + """Raised when input source code fails all parse attempts.""" class WriteBack(Enum): @@ -115,10 +116,16 @@ class FileMode(Flag): PYTHON36 = 1 PYI = 2 NO_STRING_NORMALIZATION = 4 + NO_NUMERIC_UNDERSCORE_NORMALIZATION = 8 @classmethod def from_configuration( - cls, *, py36: bool, pyi: bool, skip_string_normalization: bool + cls, + *, + py36: bool, + pyi: bool, + skip_string_normalization: bool, + skip_numeric_underscore_normalization: bool, ) -> "FileMode": mode = cls.AUTO_DETECT if py36: @@ -127,6 +134,8 @@ class FileMode(Flag): mode |= cls.PYI if skip_string_normalization: mode |= cls.NO_STRING_NORMALIZATION + if skip_numeric_underscore_normalization: + mode |= cls.NO_NUMERIC_UNDERSCORE_NORMALIZATION return mode @@ -151,7 +160,9 @@ def read_pyproject_toml( pyproject_toml = toml.load(value) config = pyproject_toml.get("tool", {}).get("black", {}) except (toml.TomlDecodeError, OSError) as e: - raise click.BadOptionUsage(f"Error reading configuration file: {e}", ctx) + raise click.FileError( + filename=value, hint=f"Error reading configuration file: {e}" + ) if not config: return None @@ -196,6 +207,12 @@ def read_pyproject_toml( is_flag=True, help="Don't normalize string quotes or prefixes.", ) +@click.option( + "-N", + "--skip-numeric-underscore-normalization", + is_flag=True, + help="Don't normalize underscores in numeric literals.", +) @click.option( "--check", is_flag=True, @@ -286,6 +303,7 @@ def main( pyi: bool, py36: bool, skip_string_normalization: bool, + skip_numeric_underscore_normalization: bool, quiet: bool, verbose: bool, include: str, @@ -296,7 +314,10 @@ def main( """The uncompromising code formatter.""" write_back = WriteBack.from_configuration(check=check, diff=diff) mode = FileMode.from_configuration( - py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization + py36=py36, + pyi=pyi, + skip_string_normalization=skip_string_normalization, + skip_numeric_underscore_normalization=skip_numeric_underscore_normalization, ) if config and verbose: out(f"Using configuration from {config}.", bold=False, fg="blue") @@ -607,7 +628,7 @@ def format_str( `line_length` determines how many characters per line are allowed. """ - src_node = lib2to3_parse(src_contents) + src_node = lib2to3_parse(src_contents.lstrip()) dst_contents = "" future_imports = get_future_imports(src_node) is_pyi = bool(mode & FileMode.PYI) @@ -618,7 +639,8 @@ def format_str( remove_u_prefix=py36 or "unicode_literals" in future_imports, is_pyi=is_pyi, normalize_strings=normalize_strings, - allow_underscores=py36, + allow_underscores=py36 + and not bool(mode & FileMode.NO_NUMERIC_UNDERSCORE_NORMALIZATION), ) elt = EmptyLineTracker(is_pyi=is_pyi) empty_line = Line() @@ -660,7 +682,6 @@ GRAMMARS = [ def lib2to3_parse(src_txt: str) -> Node: """Given a string with source, return the lib2to3 Node.""" - grammar = pygram.python_grammar_no_print_statement if src_txt[-1:] != "\n": src_txt += "\n" for grammar in GRAMMARS: @@ -676,7 +697,7 @@ def lib2to3_parse(src_txt: str) -> Node: faulty_line = lines[lineno - 1] except IndexError: faulty_line = "" - exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") + exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}") else: raise exc from None @@ -756,9 +777,7 @@ class DebugVisitor(Visitor[T]): list(v.visit(code)) -KEYWORDS = set(keyword.kwlist) WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE} -FLOW_CONTROL = {"return", "raise", "break", "continue"} STATEMENT = { syms.if_stmt, syms.while_stmt, @@ -1007,7 +1026,9 @@ class Line: depth: int = 0 leaves: List[Leaf] = Factory(list) - comments: List[Tuple[Index, Leaf]] = Factory(list) + # The LeafID keys of comments must remain ordered by the corresponding leaf's index + # in leaves + comments: Dict[LeafID, List[Leaf]] = Factory(dict) bracket_tracker: BracketTracker = Factory(BracketTracker) inside_brackets: bool = False should_explode: bool = False @@ -1214,43 +1235,35 @@ class Line: if comment.type != token.COMMENT: return False - after = len(self.leaves) - 1 - if after == -1: + if not self.leaves: comment.type = STANDALONE_COMMENT comment.prefix = "" return False else: - self.comments.append((after, comment)) - return True - - def comments_after(self, leaf: Leaf, _index: int = -1) -> Iterator[Leaf]: - """Generate comments that should appear directly after `leaf`. - - Provide a non-negative leaf `_index` to speed up the function. - """ - if not self.comments: - return - - if _index == -1: - for _index, _leaf in enumerate(self.leaves): - if leaf is _leaf: - break - + leaf_id = id(self.leaves[-1]) + if leaf_id not in self.comments: + self.comments[leaf_id] = [comment] else: - return + self.comments[leaf_id].append(comment) + return True - for index, comment_after in self.comments: - if _index == index: - yield comment_after + def comments_after(self, leaf: Leaf) -> List[Leaf]: + """Generate comments that should appear directly after `leaf`.""" + return self.comments.get(id(leaf), []) def remove_trailing_comma(self) -> None: """Remove the trailing comma and moves the comments attached to it.""" - comma_index = len(self.leaves) - 1 - for i in range(len(self.comments)): - comment_index, comment = self.comments[i] - if comment_index == comma_index: - self.comments[i] = (comma_index - 1, comment) + # Remember, the LeafID keys of self.comments are ordered by the + # corresponding leaf's index in self.leaves + # If id(self.leaves[-2]) is in self.comments, the order doesn't change. + # Otherwise, we insert it into self.comments, and it becomes the last entry. + # However, since we delete id(self.leaves[-1]) from self.comments, the invariant + # is maintained + self.comments.setdefault(id(self.leaves[-2]), []).extend( + self.comments.get(id(self.leaves[-1]), []) + ) + self.comments.pop(id(self.leaves[-1]), None) self.leaves.pop() def is_complex_subscript(self, leaf: Leaf) -> bool: @@ -1282,7 +1295,7 @@ class Line: res = f"{first.prefix}{indent}{first.value}" for leaf in leaves: res += str(leaf) - for _, comment in self.comments: + for comment in itertools.chain.from_iterable(self.comments.values()): res += str(comment) return res + "\n" @@ -1593,7 +1606,7 @@ BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT} -def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa C901 +def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 """Return whitespace prefix if needed for the given `leaf`. `complex_subscript` signals whether the given leaf is part of a subscription @@ -1874,7 +1887,7 @@ def container_of(leaf: Leaf) -> LN: return container -def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int: +def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int: """Return the priority of the `leaf` delimiter, given a line break after it. The delimiter priorities returned here are from those delimiters that would @@ -1888,7 +1901,7 @@ def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int: return 0 -def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int: +def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int: """Return the priority of the `leaf` delimiter, given a line break before it. The delimiter priorities returned here are from those delimiters that would @@ -2013,6 +2026,16 @@ def generate_comments(leaf: LN) -> Iterator[Leaf]: @dataclass class ProtoComment: + """Describes a piece of syntax that is a comment. + + It's not a :class:`blib2to3.pytree.Leaf` so that: + + * it can be cached (`Leaf` objects should not be reused more than once as + they store their lineno, column, prefix, and parent information); + * `newlines` and `consumed` fields are kept separate from the `value`. This + simplifies handling of special marker comments like ``# fmt: off/on``. + """ + type: int # token.COMMENT or STANDALONE_COMMENT value: str # content of the comment newlines: int # how many newlines before the comment @@ -2021,6 +2044,7 @@ class ProtoComment: @lru_cache(maxsize=4096) def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: + """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" result: List[ProtoComment] = [] if not prefix or "#" not in prefix: return result @@ -2052,8 +2076,8 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: def make_comment(content: str) -> str: """Return a consistently formatted comment from the given `content` string. - All comments (except for "##", "#!", "#:") should have a single space between - the hash sign and the content. + All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single + space between the hash sign and the content. If `content` didn't start with a hash sign, one is provided. """ @@ -2063,7 +2087,7 @@ def make_comment(content: str) -> str: if content[0] == "#": content = content[1:] - if content and content[0] not in " !:#": + if content and content[0] not in " !:#'%": content = " " + content return "#" + content @@ -2086,8 +2110,19 @@ def split_line( return line_str = str(line).strip("\n") - if not line.should_explode and is_line_short_enough( - line, line_length=line_length, line_str=line_str + + # we don't want to split special comments like type annotations + # https://github.com/python/typing/issues/186 + has_special_comment = False + for leaf in line.leaves: + for comment in line.comments_after(leaf): + if leaf.type == token.COMMA and is_special_comment(comment): + has_special_comment = True + + if ( + not has_special_comment + and not line.should_explode + and is_line_short_enough(line, line_length=line_length, line_str=line_str) ): yield line return @@ -2126,7 +2161,7 @@ def split_line( result.extend( split_line(l, line_length=line_length, inner=True, py36=py36) ) - except CannotSplit as cs: + except CannotSplit: continue else: @@ -2144,9 +2179,6 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: Prefer RHS otherwise. This is why this function is not symmetrical with :func:`right_hand_split` which also handles optional parentheses. """ - head = Line(depth=line.depth) - body = Line(depth=line.depth + 1, inside_brackets=True) - tail = Line(depth=line.depth) tail_leaves: List[Leaf] = [] body_leaves: List[Leaf] = [] head_leaves: List[Leaf] = [] @@ -2164,15 +2196,12 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: if leaf.type in OPENING_BRACKETS: matching_bracket = leaf current_leaves = body_leaves - # Since body is a new indent level, remove spurious leading whitespace. - if body_leaves: - normalize_prefix(body_leaves[0], inside_brackets=True) - # Build the new lines. - for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves): - for leaf in leaves: - result.append(leaf, preformatted=True) - for comment_after in line.comments_after(leaf): - result.append(comment_after, preformatted=True) + if not matching_bracket: + raise CannotSplit("No brackets found") + + head = bracket_split_build_line(head_leaves, line, matching_bracket) + body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True) + tail = bracket_split_build_line(tail_leaves, line, matching_bracket) bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): if result: @@ -2190,9 +2219,6 @@ def right_hand_split( Note: running this function modifies `bracket_depth` on the leaves of `line`. """ - head = Line(depth=line.depth) - body = Line(depth=line.depth + 1, inside_brackets=True) - tail = Line(depth=line.depth) tail_leaves: List[Leaf] = [] body_leaves: List[Leaf] = [] head_leaves: List[Leaf] = [] @@ -2209,25 +2235,18 @@ def right_hand_split( opening_bracket = leaf.opening_bracket closing_bracket = leaf current_leaves = body_leaves - tail_leaves.reverse() - body_leaves.reverse() - head_leaves.reverse() - # Since body is a new indent level, remove spurious leading whitespace. - if body_leaves: - normalize_prefix(body_leaves[0], inside_brackets=True) - if not head_leaves: - # No `head` means the split failed. Either `tail` has all content or + if not (opening_bracket and closing_bracket and head_leaves): + # If there is no opening or closing_bracket that means the split failed and + # all content is in the tail. Otherwise, if `head_leaves` are empty, it means # the matching `opening_bracket` wasn't available on `line` anymore. raise CannotSplit("No brackets found") - # Build the new lines. - for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves): - for leaf in leaves: - result.append(leaf, preformatted=True) - for comment_after in line.comments_after(leaf): - result.append(comment_after, preformatted=True) - assert opening_bracket and closing_bracket - body.should_explode = should_explode(body, opening_bracket) + tail_leaves.reverse() + body_leaves.reverse() + head_leaves.reverse() + head = bracket_split_build_line(head_leaves, line, opening_bracket) + body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True) + tail = bracket_split_build_line(tail_leaves, line, opening_bracket) bracket_split_succeeded_or_raise(head, body, tail) if ( # the body shouldn't be exploded @@ -2301,6 +2320,35 @@ def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None ) +def bracket_split_build_line( + leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False +) -> Line: + """Return a new line with given `leaves` and respective comments from `original`. + + If `is_body` is True, the result line is one-indented inside brackets and as such + has its first leaf's prefix normalized and a trailing comma added when expected. + """ + result = Line(depth=original.depth) + if is_body: + result.inside_brackets = True + result.depth += 1 + if leaves: + # Since body is a new indent level, remove spurious leading whitespace. + normalize_prefix(leaves[0], inside_brackets=True) + # Ensure a trailing comma when expected. + if original.is_import: + if leaves[-1].type != token.COMMA: + leaves.append(Leaf(token.COMMA, ",")) + # Populate the line + for leaf in leaves: + result.append(leaf, preformatted=True) + for comment_after in original.comments_after(leaf): + result.append(comment_after, preformatted=True) + if is_body: + result.should_explode = should_explode(result, opening_bracket) + return result + + def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc: """Normalize prefix of the first leaf in every line returned by `split_func`. @@ -2347,16 +2395,16 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: nonlocal current_line try: current_line.append_safe(leaf, preformatted=True) - except ValueError as ve: + except ValueError: yield current_line current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) current_line.append(leaf) - for index, leaf in enumerate(line.leaves): + for leaf in line.leaves: yield from append_to_line(leaf) - for comment_after in line.comments_after(leaf, index): + for comment_after in line.comments_after(leaf): yield from append_to_line(comment_after) lowest_depth = min(lowest_depth, leaf.bracket_depth) @@ -2393,16 +2441,16 @@ def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]: nonlocal current_line try: current_line.append_safe(leaf, preformatted=True) - except ValueError as ve: + except ValueError: yield current_line current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets) current_line.append(leaf) - for index, leaf in enumerate(line.leaves): + for leaf in line.leaves: yield from append_to_line(leaf) - for comment_after in line.comments_after(leaf, index): + for comment_after in line.comments_after(leaf): yield from append_to_line(comment_after) if current_line: @@ -2423,6 +2471,16 @@ def is_import(leaf: Leaf) -> bool: ) +def is_special_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a special comment. + Only returns true for type comments for now.""" + t = leaf.type + v = leaf.value + return bool( + (t == token.COMMENT or t == STANDALONE_COMMENT) and (v.startswith("# type:")) + ) + + def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: """Leave existing extra newlines if not `inside_brackets`. Remove everything else. @@ -2531,9 +2589,13 @@ def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None: in Python 2 long literals), and long number literals are split using underscores. """ text = leaf.value.lower() - if text.startswith(("0o", "0x", "0b")): - # Leave octal, hex, and binary literals alone. + if text.startswith(("0o", "0b")): + # Leave octal and binary literals alone. pass + elif text.startswith("0x"): + # Change hex literals to upper case. + before, after = text[:2], text[2:] + text = f"{before}{after.upper()}" elif "e" in text: before, after = text.split("e") sign = "" @@ -2585,8 +2647,8 @@ def format_int_string( return text text = text.replace("_", "") - if len(text) <= 6: - # No underscores for numbers <= 6 digits long. + if len(text) <= 5: + # No underscores for numbers <= 5 digits long. return text if count_from_end: @@ -2908,6 +2970,7 @@ def ensure_visible(leaf: Leaf) -> None: def should_explode(line: Line, opening_bracket: Leaf) -> bool: """Should `line` immediately be split with `delimiter_split()` after RHS?""" + if not ( opening_bracket.parent and opening_bracket.parent.type in {syms.atom, syms.import_from} @@ -2976,7 +3039,6 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf length = 4 * line.depth opening_bracket = None closing_bracket = None - optional_brackets: Set[LeafID] = set() inner_brackets: Set[LeafID] = set() for index, leaf, leaf_length in enumerate_with_length(line, reversed=True): length += leaf_length @@ -2987,17 +3049,12 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf if leaf.type == STANDALONE_COMMENT or has_inline_comment: break - optional_brackets.discard(id(leaf)) if opening_bracket: if leaf is opening_bracket: opening_bracket = None elif leaf.type in CLOSING_BRACKETS: inner_brackets.add(id(leaf)) elif leaf.type in CLOSING_BRACKETS: - if not leaf.value: - optional_brackets.add(id(opening_bracket)) - continue - if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS: # Empty brackets would fail a split so treat them as "inner" # brackets (e.g. only add them to the `omit` set if another @@ -3005,13 +3062,15 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf inner_brackets.add(id(leaf)) continue - opening_bracket = leaf.opening_bracket if closing_bracket: omit.add(id(closing_bracket)) omit.update(inner_brackets) inner_brackets.clear() yield omit - closing_bracket = leaf + + if leaf.value: + opening_bracket = leaf.opening_bracket + closing_bracket = leaf def get_future_imports(node: Node) -> Set[str]: @@ -3394,7 +3453,7 @@ def enumerate_with_length( return # Multiline strings, we can't continue. comment: Optional[Leaf] - for comment in line.comments_after(leaf, index): + for comment in line.comments_after(leaf): length += len(comment.value) yield index, leaf, length @@ -3589,11 +3648,11 @@ def write_cache( """Update the cache file.""" cache_file = get_cache_file(line_length, mode) try: - if not CACHE_DIR.exists(): - CACHE_DIR.mkdir(parents=True) + CACHE_DIR.mkdir(parents=True, exist_ok=True) new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}} - with cache_file.open("wb") as fobj: - pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL) + with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: + pickle.dump(new_cache, f, protocol=pickle.HIGHEST_PROTOCOL) + os.replace(f.name, cache_file) except OSError: pass @@ -3620,6 +3679,11 @@ def patch_click() -> None: module._verify_python3_env = lambda: None -if __name__ == "__main__": +def patched_main() -> None: + freeze_support() patch_click() main() + + +if __name__ == "__main__": + patched_main()