X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/b1a7600b0a0691821926a07015f7bc66ca0ae4e9..b901d75deb497aa56a1d7651a3386af8a495afc9:/black.py diff --git a/black.py b/black.py index 81efa59..de86156 100644 --- a/black.py +++ b/black.py @@ -10,7 +10,7 @@ from pathlib import Path import tokenize import sys from typing import ( - Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union + Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union ) from attr import dataclass, Factory @@ -22,7 +22,7 @@ from blib2to3 import pygram, pytree from blib2to3.pgen2 import driver, token from blib2to3.pgen2.parse import ParseError -__version__ = "18.3a2" +__version__ = "18.3a4" DEFAULT_LINE_LENGTH = 88 # types syms = pygram.python_symbols @@ -38,16 +38,44 @@ err = partial(click.secho, fg='red', err=True) class NothingChanged(UserWarning): - """Raised by `format_file` when the reformatted code is the same as source.""" + """Raised by `format_file()` when the reformatted code is the same as source.""" class CannotSplit(Exception): """A readable split that fits the allotted line length is impossible. - Raised by `left_hand_split()` and `right_hand_split()`. + Raised by `left_hand_split()`, `right_hand_split()`, and `delimiter_split()`. """ +class FormatError(Exception): + """Base exception for `# fmt: on` and `# fmt: off` handling. + + It holds the number of bytes of the prefix consumed before the format + control comment appeared. + """ + + def __init__(self, consumed: int) -> None: + super().__init__(consumed) + self.consumed = consumed + + def trim_prefix(self, leaf: Leaf) -> None: + leaf.prefix = leaf.prefix[self.consumed:] + + def leaf_from_consumed(self, leaf: Leaf) -> Leaf: + """Returns a new Leaf from the consumed part of the prefix.""" + unformatted_prefix = leaf.prefix[:self.consumed] + return Leaf(token.NEWLINE, unformatted_prefix) + + +class FormatOn(FormatError): + """Found a comment like `# fmt: on` in the file.""" + + +class FormatOff(FormatError): + """Found a comment like `# fmt: off` in the file.""" + + @click.command() @click.option( '-l', @@ -62,7 +90,7 @@ class CannotSplit(Exception): is_flag=True, help=( "Don't write back the files, just return the status. Return code 0 " - "means nothing changed. Return code 1 means some files were " + "means nothing would change. Return code 1 means some files would be " "reformatted. Return code 123 means there was an internal error." ), ) @@ -100,7 +128,7 @@ def main( ctx.exit(0) elif len(sources) == 1: p = sources[0] - report = Report() + report = Report(check=check) try: if not p.is_file() and str(p) == '-': changed = format_stdin_to_stdout( @@ -137,6 +165,13 @@ async def schedule_formatting( loop: BaseEventLoop, executor: Executor, ) -> int: + """Run formatting of `sources` in parallel using the provided `executor`. + + (Use ProcessPoolExecutors for actual parallelism.) + + `line_length`, `write_back`, and `fast` options are passed to + :func:`format_file_in_place`. + """ tasks = { src: loop.run_in_executor( executor, format_file_in_place, src, line_length, fast, write_back @@ -165,7 +200,11 @@ async def schedule_formatting( def format_file_in_place( src: Path, line_length: int, fast: bool, write_back: bool = False ) -> bool: - """Format the file and rewrite if changed. Return True if changed.""" + """Format file under `src` path. Return True if changed. + + If `write_back` is True, write reformatted code back to stdout. + `line_length` and `fast` options are passed to :func:`format_file_contents`. + """ with tokenize.open(src) as src_buffer: src_contents = src_buffer.read() try: @@ -184,7 +223,11 @@ def format_file_in_place( def format_stdin_to_stdout( line_length: int, fast: bool, write_back: bool = False ) -> bool: - """Format file on stdin and pipe output to stdout. Return True if changed.""" + """Format file on stdin. Return True if changed. + + If `write_back` is True, write reformatted code back to stdout. + `line_length` and `fast` arguments are passed to :func:`format_file_contents`. + """ contents = sys.stdin.read() try: contents = format_file_contents(contents, line_length=line_length, fast=fast) @@ -201,7 +244,12 @@ def format_stdin_to_stdout( def format_file_contents( src_contents: str, line_length: int, fast: bool ) -> FileContent: - """Reformats a file and returns its contents and encoding.""" + """Reformats a file and returns its contents and encoding. + + If `fast` is False, additionally confirm that the reformatted code is + valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it. + `line_length` is passed to :func:`format_str`. + """ if src_contents.strip() == '': raise NothingChanged @@ -216,7 +264,10 @@ def format_file_contents( def format_str(src_contents: str, line_length: int) -> FileContent: - """Reformats a string and returns new contents.""" + """Reformats a string and returns new contents. + + `line_length` determines how many characters per line are allowed. + """ src_node = lib2to3_parse(src_contents) dst_contents = "" lines = LineGenerator() @@ -235,23 +286,36 @@ def format_str(src_contents: str, line_length: int) -> FileContent: return dst_contents +GRAMMARS = [ + pygram.python_grammar_no_print_statement_no_exec_statement, + pygram.python_grammar_no_print_statement, + pygram.python_grammar_no_exec_statement, + pygram.python_grammar, +] + + def lib2to3_parse(src_txt: str) -> Node: """Given a string with source, return the lib2to3 Node.""" grammar = pygram.python_grammar_no_print_statement - drv = driver.Driver(grammar, pytree.convert) if src_txt[-1] != '\n': nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n' src_txt += nl - try: - result = drv.parse_string(src_txt, True) - except ParseError as pe: - lineno, column = pe.context[1] - lines = src_txt.splitlines() + for grammar in GRAMMARS: + drv = driver.Driver(grammar, pytree.convert) try: - faulty_line = lines[lineno - 1] - except IndexError: - faulty_line = "" - raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None + result = drv.parse_string(src_txt, True) + break + + except ParseError as pe: + lineno, column = pe.context[1] + lines = src_txt.splitlines() + try: + faulty_line = lines[lineno - 1] + except IndexError: + faulty_line = "" + exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") + else: + raise exc from None if isinstance(result, Leaf): result = Node(syms.file_input, [result]) @@ -268,9 +332,18 @@ T = TypeVar('T') class Visitor(Generic[T]): - """Basic lib2to3 visitor that yields things on visiting.""" + """Basic lib2to3 visitor that yields things of type `T` on `visit()`.""" def visit(self, node: LN) -> Iterator[T]: + """Main method to visit `node` and its children. + + It tries to find a `visit_*()` method for the given `node.type`, like + `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects. + If no dedicated `visit_*()` method is found, chooses `visit_default()` + instead. + + Then yields objects of type `T` from the selected visitor. + """ if node.type < 256: name = token.tok_name[node.type] else: @@ -278,6 +351,7 @@ class Visitor(Generic[T]): yield from getattr(self, f'visit_{name}', self.visit_default)(node) def visit_default(self, node: LN) -> Iterator[T]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" if isinstance(node, Node): for child in node.children: yield from self.visit(child) @@ -307,6 +381,15 @@ class DebugVisitor(Visitor[T]): out(f' {node.prefix!r}', fg='green', bold=False, nl=False) out(f' {node.value!r}', fg='blue', bold=False) + @classmethod + def show(cls, code: str) -> None: + """Pretty-prints a given string of `code`. + + Convenience method for debugging. + """ + v: DebugVisitor[None] = DebugVisitor() + list(v.visit(lib2to3_parse(code))) + KEYWORDS = set(keyword.kwlist) WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE} @@ -340,6 +423,7 @@ MATH_OPERATORS = { token.AMPER, token.PERCENT, token.CIRCUMFLEX, + token.TILDE, token.LEFTSHIFT, token.RIGHTSHIFT, token.DOUBLESTAR, @@ -355,12 +439,28 @@ MATH_PRIORITY = 1 @dataclass class BracketTracker: + """Keeps track of brackets on a line.""" + depth: int = 0 bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict) delimiters: Dict[LeafID, Priority] = Factory(dict) previous: Optional[Leaf] = None def mark(self, leaf: Leaf) -> None: + """Marks `leaf` with bracket-related metadata. Keeps track of delimiters. + + All leaves receive an int `bracket_depth` field that stores how deep + within brackets a given leaf is. 0 means there are no enclosing brackets + that started on this line. + + If a leaf is itself a closing bracket, it receives an `opening_bracket` + field that it forms a pair with. This is a one-directional link to + avoid reference cycles. + + If a leaf is a delimiter (a token on which Black can split the line if + needed) and it's on depth 0, its `id()` is stored in the tracker's + `delimiters` field. + """ if leaf.type == token.COMMENT: return @@ -405,7 +505,7 @@ class BracketTracker: """Returns True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) - def max_priority(self, exclude: Iterable[LeafID] =()) -> int: + def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int: """Returns the highest priority of a delimiter found on the line. Values are consistent with what `is_delimiter()` returns. @@ -415,6 +515,8 @@ class BracketTracker: @dataclass class Line: + """Holds leaves and comments. Can be printed with `str(line)`.""" + depth: int = 0 leaves: List[Leaf] = Factory(list) comments: Dict[LeafID, Leaf] = Factory(dict) @@ -424,6 +526,15 @@ class Line: _for_loop_variable: bool = False def append(self, leaf: Leaf, preformatted: bool = False) -> None: + """Add a new `leaf` to the end of the line. + + Unless `preformatted` is True, the `leaf` will receive a new consistent + whitespace prefix and metadata applied by :class:`BracketTracker`. + Trailing commas are maybe removed, unpacked for loop variables are + demoted from being delimiters. + + Inline comments are put aside. + """ has_value = leaf.value.strip() if not has_value: return @@ -445,18 +556,22 @@ class Line: @property def is_comment(self) -> bool: + """Is this line a standalone comment?""" return bool(self) and self.leaves[0].type == STANDALONE_COMMENT @property def is_decorator(self) -> bool: + """Is this line a decorator?""" return bool(self) and self.leaves[0].type == token.AT @property def is_import(self) -> bool: + """Is this an import line?""" return bool(self) and is_import(self.leaves[0]) @property def is_class(self) -> bool: + """Is this a class definition?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -465,7 +580,7 @@ class Line: @property def is_def(self) -> bool: - """Also returns True for async defs.""" + """Is this a function definition? (Also returns True for async defs.)""" try: first_leaf = self.leaves[0] except IndexError: @@ -487,6 +602,10 @@ class Line: @property def is_flow_control(self) -> bool: + """Is this a flow control statement? + + Those are `return`, `raise`, `break`, and `continue`. + """ return ( bool(self) and self.leaves[0].type == token.NAME @@ -495,6 +614,7 @@ class Line: @property def is_yield(self) -> bool: + """Is this a yield statement?""" return ( bool(self) and self.leaves[0].type == token.NAME @@ -502,6 +622,7 @@ class Line: ) def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: + """Remove trailing comma if there is one and it's safe.""" if not ( self.leaves and self.leaves[-1].type == token.COMMA @@ -509,10 +630,16 @@ class Line: ): return False - if closing.type == token.RSQB or closing.type == token.RBRACE: + if closing.type == token.RBRACE: self.leaves.pop() return True + if closing.type == token.RSQB: + comma = self.leaves[-1] + if comma.parent and comma.parent.type == syms.listmaker: + self.leaves.pop() + return True + # For parens let's check if it's safe to remove the comma. If the # trailing one is the only one, we might mistakenly change a tuple # into a different type by removing the comma. @@ -558,7 +685,7 @@ class Line: return False def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool: - # See `maybe_increment_for_loop_variable` above for explanation. + """See `maybe_increment_for_loop_variable` above for explanation.""" if self._for_loop_variable and leaf.type == token.NAME and leaf.value == 'in': self.bracket_tracker.depth -= 1 self._for_loop_variable = False @@ -586,6 +713,7 @@ class Line: return self.append_comment(comment) def append_comment(self, comment: Leaf) -> bool: + """Add an inline comment to the line.""" if comment.type != token.COMMENT: return False @@ -604,6 +732,7 @@ class Line: return True def last_non_delimiter(self) -> Leaf: + """Returns the last non-delimiter on the line. Raises LookupError otherwise.""" for i in range(len(self.leaves)): last = self.leaves[-i - 1] if not is_delimiter(last): @@ -612,6 +741,7 @@ class Line: raise LookupError("No non-delimiters found") def __str__(self) -> str: + """Render the line.""" if not self: return '\n' @@ -626,9 +756,63 @@ class Line: return res + '\n' def __bool__(self) -> bool: + """Returns True if the line has leaves or comments.""" return bool(self.leaves or self.comments) +class UnformattedLines(Line): + """Just like :class:`Line` but stores lines which aren't reformatted.""" + + def append(self, leaf: Leaf, preformatted: bool = True) -> None: + """Just add a new `leaf` to the end of the lines. + + The `preformatted` argument is ignored. + + Keeps track of indentation `depth`, which is useful when the user + says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`. + """ + try: + list(generate_comments(leaf)) + except FormatOn as f_on: + self.leaves.append(f_on.leaf_from_consumed(leaf)) + raise + + self.leaves.append(leaf) + if leaf.type == token.INDENT: + self.depth += 1 + elif leaf.type == token.DEDENT: + self.depth -= 1 + + def append_comment(self, comment: Leaf) -> bool: + """Not implemented in this class.""" + raise NotImplementedError("Unformatted lines don't store comments separately.") + + def maybe_remove_trailing_comma(self, closing: Leaf) -> bool: + """Does nothing and returns False.""" + return False + + def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool: + """Does nothing and returns False.""" + return False + + def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool: + """Does nothing and returns False.""" + return False + + def __str__(self) -> str: + """Renders unformatted lines from leaves which were added with `append()`. + + `depth` is not used for indentation in this case. + """ + if not self: + return '\n' + + res = '' + for leaf in self.leaves: + res += str(leaf) + return res + + @dataclass class EmptyLineTracker: """Provides a stateful method that returns the number of potential extra @@ -649,6 +833,9 @@ class EmptyLineTracker: (two on module-level), as well as providing an extra empty line after flow control keywords to make them more prominent. """ + if isinstance(current_line, UnformattedLines): + return 0, 0 + before, after = self._maybe_empty_lines(current_line) before -= self.previous_after self.previous_after = after @@ -657,13 +844,13 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: max_allowed = 1 - if current_line.is_comment and current_line.depth == 0: + if current_line.depth == 0: max_allowed = 2 if current_line.leaves: # Consume the first leaf's extra newlines. first_leaf = current_line.leaves[0] before = first_leaf.prefix.count('\n') - before = min(before, max(before, max_allowed)) + before = min(before, max_allowed) first_leaf.prefix = '' else: before = 0 @@ -718,7 +905,7 @@ class LineGenerator(Visitor[Line]): """ current_line: Line = Factory(Line) - def line(self, indent: int = 0) -> Iterator[Line]: + def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]: """Generate a line. If the line is empty, only emit if it makes sense. @@ -727,49 +914,82 @@ class LineGenerator(Visitor[Line]): If any lines were generated, set up a new current_line. """ if not self.current_line: - self.current_line.depth += indent + if self.current_line.__class__ == type: + self.current_line.depth += indent + else: + self.current_line = type(depth=self.current_line.depth + indent) return # Line is empty, don't emit. Creating a new one unnecessary. complete_line = self.current_line - self.current_line = Line(depth=complete_line.depth + indent) + self.current_line = type(depth=complete_line.depth + indent) yield complete_line + def visit(self, node: LN) -> Iterator[Line]: + """Main method to start the visit process. Yields :class:`Line` objects.""" + if isinstance(self.current_line, UnformattedLines): + # File contained `# fmt: off` + yield from self.visit_unformatted(node) + + else: + yield from super().visit(node) + def visit_default(self, node: LN) -> Iterator[Line]: + """Default `visit_*()` implementation. Recurses to children of `node`.""" if isinstance(node, Leaf): any_open_brackets = self.current_line.bracket_tracker.any_open_brackets() - for comment in generate_comments(node): - if any_open_brackets: - # any comment within brackets is subject to splitting - self.current_line.append(comment) - elif comment.type == token.COMMENT: - # regular trailing comment - self.current_line.append(comment) - yield from self.line() - - else: - # regular standalone comment - yield from self.line() - - self.current_line.append(comment) - yield from self.line() - - normalize_prefix(node, inside_brackets=any_open_brackets) - if node.type not in WHITESPACE: - self.current_line.append(node) + try: + for comment in generate_comments(node): + if any_open_brackets: + # any comment within brackets is subject to splitting + self.current_line.append(comment) + elif comment.type == token.COMMENT: + # regular trailing comment + self.current_line.append(comment) + yield from self.line() + + else: + # regular standalone comment + yield from self.line() + + self.current_line.append(comment) + yield from self.line() + + except FormatOff as f_off: + f_off.trim_prefix(node) + yield from self.line(type=UnformattedLines) + yield from self.visit(node) + + except FormatOn as f_on: + # This only happens here if somebody says "fmt: on" multiple + # times in a row. + f_on.trim_prefix(node) + yield from self.visit_default(node) + + else: + normalize_prefix(node, inside_brackets=any_open_brackets) + if node.type not in WHITESPACE: + self.current_line.append(node) yield from super().visit_default(node) def visit_INDENT(self, node: Node) -> Iterator[Line]: + """Increases indentation level, maybe yields a line.""" + # In blib2to3 INDENT never holds comments. yield from self.line(+1) yield from self.visit_default(node) def visit_DEDENT(self, node: Node) -> Iterator[Line]: + """Decreases indentation level, maybe yields a line.""" + # DEDENT has no value. Additionally, in blib2to3 it never holds comments. yield from self.line(-1) def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]: - """Visit a statement. + """Visits a statement. - The relevant Python language keywords for this statement are NAME leaves - within it. + This implementation is shared for `if`, `while`, `for`, `try`, `except`, + `def`, `with`, and `class`. + + The relevant Python language `keywords` for a given statement will be NAME + leaves within it. This methods puts those on a separate line. """ for child in node.children: if child.type == token.NAME and child.value in keywords: # type: ignore @@ -778,7 +998,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_simple_stmt(self, node: Node) -> Iterator[Line]: - """A statement without nested statements.""" + """Visits a statement without nested statements.""" is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: yield from self.line(+1) @@ -790,6 +1010,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) def visit_async_stmt(self, node: Node) -> Iterator[Line]: + """Visits `async def`, `async for`, `async with`.""" yield from self.line() children = iter(node.children) @@ -804,17 +1025,40 @@ class LineGenerator(Visitor[Line]): yield from self.visit(child) def visit_decorators(self, node: Node) -> Iterator[Line]: + """Visits decorators.""" for child in node.children: yield from self.line() yield from self.visit(child) def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]: + """Semicolons are always removed. + + Statements between them are put on separate lines. + """ yield from self.line() def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]: + """End of file. + + Process outstanding comments and end with a newline. + """ yield from self.visit_default(leaf) yield from self.line() + def visit_unformatted(self, node: LN) -> Iterator[Line]: + """Used when file contained a `# fmt: off`.""" + if isinstance(node, Node): + for child in node.children: + yield from self.visit(child) + + else: + try: + self.current_line.append(node) + except FormatOn as f_on: + f_on.trim_prefix(node) + yield from self.line() + yield from self.visit(node) + def __attrs_post_init__(self) -> None: """You are in a twisty little maze of passages.""" v = self.visit_stmt @@ -865,22 +1109,26 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 return SPACE if prevp.type == token.COMMA else NO if prevp.type == token.EQUAL: - if prevp.parent and prevp.parent.type in { - syms.typedargslist, - syms.varargslist, - syms.parameters, - syms.arglist, - syms.argument, - }: - return NO + if prevp.parent: + if prevp.parent.type in { + syms.arglist, syms.argument, syms.parameters, syms.varargslist + }: + return NO + + elif prevp.parent.type == syms.typedargslist: + # A bit hacky: if the equal sign has whitespace, it means we + # previously found it's a typed argument. So, we're using + # that, too. + return prevp.prefix elif prevp.type == token.DOUBLESTAR: if prevp.parent and prevp.parent.type in { - syms.typedargslist, - syms.varargslist, - syms.parameters, syms.arglist, + syms.argument, syms.dictsetmaker, + syms.parameters, + syms.typedargslist, + syms.varargslist, }: return NO @@ -888,7 +1136,22 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}: return NO - elif prevp.parent and prevp.parent.type in {syms.factor, syms.star_expr}: + elif ( + prevp.parent + and prevp.parent.type in {syms.factor, syms.star_expr} + and prevp.type in MATH_OPERATORS + ): + return NO + + elif ( + prevp.type == token.RIGHTSHIFT + and prevp.parent + and prevp.parent.type == syms.shift_expr + and prevp.prev_sibling + and prevp.prev_sibling.type == token.NAME + and prevp.prev_sibling.value == 'print' # type: ignore + ): + # Python 2 print chevron return NO elif prev.type in OPENING_BRACKETS: @@ -902,7 +1165,7 @@ def whitespace(leaf: Leaf) -> str: # noqa C901 if not prev or prev.type != token.COMMA: return NO - if p.type == syms.varargslist: + elif p.type == syms.varargslist: # lambdas if t == token.RPAR: return NO @@ -1120,8 +1383,10 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]: if '#' not in p: return + consumed = 0 nlines = 0 for index, line in enumerate(p.split('\n')): + consumed += len(line) + 1 # adding the length of the split '\n' line = line.lstrip() if not line: nlines += 1 @@ -1132,19 +1397,33 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]: comment_type = token.COMMENT # simple trailing comment else: comment_type = STANDALONE_COMMENT - yield Leaf(comment_type, make_comment(line), prefix='\n' * nlines) + comment = make_comment(line) + yield Leaf(comment_type, comment, prefix='\n' * nlines) + + if comment in {'# fmt: on', '# yapf: enable'}: + raise FormatOn(consumed) + + if comment in {'# fmt: off', '# yapf: disable'}: + raise FormatOff(consumed) nlines = 0 def make_comment(content: str) -> str: + """Returns a consistently formatted comment from the given `content` string. + + All comments (except for "##", "#!", "#:") should have a single space between + the hash sign and the content. + + If `content` didn't start with a hash sign, one is provided. + """ content = content.rstrip() if not content: return '#' if content[0] == '#': content = content[1:] - if content and content[0] not in {' ', '!', '#'}: + if content and content[0] not in ' !:#': content = ' ' + content return '#' + content @@ -1162,6 +1441,10 @@ def split_line( If `py36` is True, splitting may generate syntax that is only compatible with Python 3.6 and later. """ + if isinstance(line, UnformattedLines): + yield line + return + line_str = str(line).strip('\n') if len(line_str) <= line_length and '\n' not in line_str: yield line @@ -1202,7 +1485,7 @@ def split_line( def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split line into many lines, starting with the first matching bracket pair. + """Splits line into many lines, starting with the first matching bracket pair. Note: this usually looks weird, only use this for function definitions. Prefer RHS otherwise. @@ -1239,14 +1522,14 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: comment_after = line.comments.get(id(leaf)) if comment_after: result.append(comment_after, preformatted=True) - split_succeeded_or_raise(head, body, tail) + bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): if result: yield result def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split line into many lines, starting with the last matching bracket pair.""" + """Splits line into many lines, starting with the last matching bracket pair.""" head = Line(depth=line.depth) body = Line(depth=line.depth + 1, inside_brackets=True) tail = Line(depth=line.depth) @@ -1279,13 +1562,26 @@ def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]: comment_after = line.comments.get(id(leaf)) if comment_after: result.append(comment_after, preformatted=True) - split_succeeded_or_raise(head, body, tail) + bracket_split_succeeded_or_raise(head, body, tail) for result in (head, body, tail): if result: yield result -def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: +def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: + """Raise :exc:`CannotSplit` if the last left- or right-hand split failed. + + Do nothing otherwise. + + A left- or right-hand split is based on a pair of brackets. Content before + (and including) the opening bracket is left on one line, content inside the + brackets is put on a separate line, and finally content starting with and + following the closing bracket is put on a separate line. + + Those are called `head`, `body`, and `tail`, respectively. If the split + produced the same line (all content in `head`) or ended up with an empty `body` + and the `tail` is just the closing bracket, then it's considered failed. + """ tail_len = len(str(tail).strip()) if not body: if tail_len == 0: @@ -1299,11 +1595,11 @@ def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None: def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: - """Split according to delimiters of the highest priority. + """Splits according to delimiters of the highest priority. This kind of split doesn't increase indentation. If `py36` is True, the split will add trailing commas also in function - signatures that contain * and **. + signatures that contain `*` and `**`. """ try: last_leaf = line.leaves[-1] @@ -1312,7 +1608,9 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]: delimiters = line.bracket_tracker.delimiters try: - delimiter_priority = line.bracket_tracker.max_priority(exclude={id(last_leaf)}) + delimiter_priority = line.bracket_tracker.max_delimiter_priority( + exclude={id(last_leaf)} + ) except ValueError: raise CannotSplit("No delimiters found") @@ -1363,9 +1661,9 @@ def is_import(leaf: Leaf) -> bool: def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leave existing extra newlines if not `inside_brackets`. + """Leaves existing extra newlines if not `inside_brackets`. - Remove everything else. Note: don't use backslashes for formatting or + Removes everything else. Note: don't use backslashes for formatting or you'll lose your voting rights. """ if not inside_brackets: @@ -1412,6 +1710,9 @@ BLACKLISTED_DIRECTORIES = { def gen_python_files_in_dir(path: Path) -> Iterator[Path]: + """Generates all files under `path` which aren't under BLACKLISTED_DIRECTORIES + and have one of the PYTHON_EXTENSIONS. + """ for child in path.iterdir(): if child.is_dir(): if child.name in BLACKLISTED_DIRECTORIES: @@ -1425,7 +1726,8 @@ def gen_python_files_in_dir(path: Path) -> Iterator[Path]: @dataclass class Report: - """Provides a reformatting counter.""" + """Provides a reformatting counter. Can be rendered with `str(report)`.""" + check: bool = False change_count: int = 0 same_count: int = 0 failure_count: int = 0 @@ -1433,7 +1735,8 @@ class Report: def done(self, src: Path, changed: bool) -> None: """Increment the counter for successful reformatting. Write out a message.""" if changed: - out(f'reformatted {src}') + reformatted = 'would reformat' if self.check else 'reformatted' + out(f'{reformatted} {src}') self.change_count += 1 else: out(f'{src} already well formatted, good job.', bold=False) @@ -1452,7 +1755,7 @@ class Report: if self.failure_count: return 123 - elif self.change_count: + elif self.change_count and self.check: return 1 return 0 @@ -1462,21 +1765,27 @@ class Report: Use `click.unstyle` to remove colors. """ + if self.check: + reformatted = "would be reformatted" + unchanged = "would be left unchanged" + failed = "would fail to reformat" + else: + reformatted = "reformatted" + unchanged = "left unchanged" + failed = "failed to reformat" report = [] if self.change_count: s = 's' if self.change_count > 1 else '' report.append( - click.style(f'{self.change_count} file{s} reformatted', bold=True) + click.style(f'{self.change_count} file{s} {reformatted}', bold=True) ) if self.same_count: s = 's' if self.same_count > 1 else '' - report.append(f'{self.same_count} file{s} left unchanged') + report.append(f'{self.same_count} file{s} {unchanged}') if self.failure_count: s = 's' if self.failure_count > 1 else '' report.append( - click.style( - f'{self.failure_count} file{s} failed to reformat', fg='red' - ) + click.style(f'{self.failure_count} file{s} {failed}', fg='red') ) return ', '.join(report) + '.' @@ -1518,7 +1827,12 @@ def assert_equivalent(src: str, dst: str) -> None: try: src_ast = ast.parse(src) except Exception as exc: - raise AssertionError(f"cannot parse source: {exc}") from None + major, minor = sys.version_info[:2] + raise AssertionError( + f"cannot use --safe with this file; failed to parse source file " + f"with Python {major}.{minor}'s builtin AST. Re-run with --fast " + f"or stop using deprecated Python 2 syntax. AST error message: {exc}" + ) try: dst_ast = ast.parse(dst)