import tokenize
import sys
from typing import (
- Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union
+ Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union
)
from attr import dataclass, Factory
from blib2to3.pgen2 import driver, token
from blib2to3.pgen2.parse import ParseError
-__version__ = "18.3a3"
+__version__ = "18.3a4"
DEFAULT_LINE_LENGTH = 88
# types
syms = pygram.python_symbols
"""
+class FormatError(Exception):
+ """Base fmt: on/off error.
+
+ It holds the number of bytes of the prefix consumed before the format
+ control comment appeared.
+ """
+
+ def __init__(self, consumed: int) -> None:
+ super().__init__(consumed)
+ self.consumed = consumed
+
+ def trim_prefix(self, leaf: Leaf) -> None:
+ leaf.prefix = leaf.prefix[self.consumed:]
+
+ def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
+ """Returns a new Leaf from the consumed part of the prefix."""
+ unformatted_prefix = leaf.prefix[:self.consumed]
+ return Leaf(token.NEWLINE, unformatted_prefix)
+
+
+class FormatOn(FormatError):
+ """Found a comment like `# fmt: on` in the file."""
+
+
+class FormatOff(FormatError):
+ """Found a comment like `# fmt: off` in the file."""
+
+
@click.command()
@click.option(
'-l',
return dst_contents
+GRAMMARS = [
+ pygram.python_grammar_no_print_statement_no_exec_statement,
+ pygram.python_grammar_no_print_statement,
+ pygram.python_grammar_no_exec_statement,
+ pygram.python_grammar,
+]
+
+
def lib2to3_parse(src_txt: str) -> Node:
"""Given a string with source, return the lib2to3 Node."""
grammar = pygram.python_grammar_no_print_statement
- drv = driver.Driver(grammar, pytree.convert)
if src_txt[-1] != '\n':
nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
src_txt += nl
- try:
- result = drv.parse_string(src_txt, True)
- except ParseError as pe:
- lineno, column = pe.context[1]
- lines = src_txt.splitlines()
+ for grammar in GRAMMARS:
+ drv = driver.Driver(grammar, pytree.convert)
try:
- faulty_line = lines[lineno - 1]
- except IndexError:
- faulty_line = "<line number missing in source>"
- raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None
+ result = drv.parse_string(src_txt, True)
+ break
+
+ except ParseError as pe:
+ lineno, column = pe.context[1]
+ lines = src_txt.splitlines()
+ try:
+ faulty_line = lines[lineno - 1]
+ except IndexError:
+ faulty_line = "<line number missing in source>"
+ exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+ else:
+ raise exc from None
if isinstance(result, Leaf):
result = Node(syms.file_input, [result])
out(f' {node.prefix!r}', fg='green', bold=False, nl=False)
out(f' {node.value!r}', fg='blue', bold=False)
+ @classmethod
+ def show(cls, code: str) -> None:
+ """Pretty-prints a given string of `code`.
+
+ Convenience method for debugging.
+ """
+ v: DebugVisitor[None] = DebugVisitor()
+ list(v.visit(lib2to3_parse(code)))
+
KEYWORDS = set(keyword.kwlist)
WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
"""Returns True if there is an yet unmatched open bracket on the line."""
return bool(self.bracket_match)
- def max_priority(self, exclude: Iterable[LeafID] =()) -> int:
+ def max_priority(self, exclude: Iterable[LeafID] = ()) -> int:
"""Returns the highest priority of a delimiter found on the line.
Values are consistent with what `is_delimiter()` returns.
return bool(self.leaves or self.comments)
+class UnformattedLines(Line):
+
+ def append(self, leaf: Leaf, preformatted: bool = False) -> None:
+ try:
+ list(generate_comments(leaf))
+ except FormatOn as f_on:
+ self.leaves.append(f_on.leaf_from_consumed(leaf))
+ raise
+
+ self.leaves.append(leaf)
+ if leaf.type == token.INDENT:
+ self.depth += 1
+ elif leaf.type == token.DEDENT:
+ self.depth -= 1
+
+ def append_comment(self, comment: Leaf) -> bool:
+ raise NotImplementedError("Unformatted lines don't store comments separately.")
+
+ def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
+ return False
+
+ def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
+ return False
+
+ def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool:
+ return False
+
+ def __str__(self) -> str:
+ if not self:
+ return '\n'
+
+ res = ''
+ for leaf in self.leaves:
+ res += str(leaf)
+ return res
+
+
@dataclass
class EmptyLineTracker:
"""Provides a stateful method that returns the number of potential extra
(two on module-level), as well as providing an extra empty line after flow
control keywords to make them more prominent.
"""
+ if isinstance(current_line, UnformattedLines):
+ return 0, 0
+
before, after = self._maybe_empty_lines(current_line)
before -= self.previous_after
self.previous_after = after
def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
max_allowed = 1
- if current_line.is_comment and current_line.depth == 0:
+ if current_line.depth == 0:
max_allowed = 2
if current_line.leaves:
# Consume the first leaf's extra newlines.
first_leaf = current_line.leaves[0]
before = first_leaf.prefix.count('\n')
- before = min(before, max(before, max_allowed))
+ before = min(before, max_allowed)
first_leaf.prefix = ''
else:
before = 0
"""
current_line: Line = Factory(Line)
- def line(self, indent: int = 0) -> Iterator[Line]:
+ def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]:
"""Generate a line.
If the line is empty, only emit if it makes sense.
If any lines were generated, set up a new current_line.
"""
if not self.current_line:
- self.current_line.depth += indent
+ if self.current_line.__class__ == type:
+ self.current_line.depth += indent
+ else:
+ self.current_line = type(depth=self.current_line.depth + indent)
return # Line is empty, don't emit. Creating a new one unnecessary.
complete_line = self.current_line
- self.current_line = Line(depth=complete_line.depth + indent)
+ self.current_line = type(depth=complete_line.depth + indent)
yield complete_line
+ def visit(self, node: LN) -> Iterator[Line]:
+ """High-level entry point to the visitor."""
+ if isinstance(self.current_line, UnformattedLines):
+ # File contained `# fmt: off`
+ yield from self.visit_unformatted(node)
+
+ else:
+ yield from super().visit(node)
+
def visit_default(self, node: LN) -> Iterator[Line]:
if isinstance(node, Leaf):
any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
- for comment in generate_comments(node):
- if any_open_brackets:
- # any comment within brackets is subject to splitting
- self.current_line.append(comment)
- elif comment.type == token.COMMENT:
- # regular trailing comment
- self.current_line.append(comment)
- yield from self.line()
-
- else:
- # regular standalone comment
- yield from self.line()
-
- self.current_line.append(comment)
- yield from self.line()
-
- normalize_prefix(node, inside_brackets=any_open_brackets)
- if node.type not in WHITESPACE:
- self.current_line.append(node)
+ try:
+ for comment in generate_comments(node):
+ if any_open_brackets:
+ # any comment within brackets is subject to splitting
+ self.current_line.append(comment)
+ elif comment.type == token.COMMENT:
+ # regular trailing comment
+ self.current_line.append(comment)
+ yield from self.line()
+
+ else:
+ # regular standalone comment
+ yield from self.line()
+
+ self.current_line.append(comment)
+ yield from self.line()
+
+ except FormatOff as f_off:
+ f_off.trim_prefix(node)
+ yield from self.line(type=UnformattedLines)
+ yield from self.visit(node)
+
+ except FormatOn as f_on:
+ # This only happens here if somebody says "fmt: on" multiple
+ # times in a row.
+ f_on.trim_prefix(node)
+ yield from self.visit_default(node)
+
+ else:
+ normalize_prefix(node, inside_brackets=any_open_brackets)
+ if node.type not in WHITESPACE:
+ self.current_line.append(node)
yield from super().visit_default(node)
def visit_INDENT(self, node: Node) -> Iterator[Line]:
yield from self.visit_default(node)
def visit_DEDENT(self, node: Node) -> Iterator[Line]:
+ # DEDENT has no value. Additionally, in blib2to3 it never holds comments.
yield from self.line(-1)
def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]:
yield from self.visit_default(leaf)
yield from self.line()
+ def visit_unformatted(self, node: LN) -> Iterator[Line]:
+ if isinstance(node, Node):
+ for child in node.children:
+ yield from self.visit(child)
+
+ else:
+ try:
+ self.current_line.append(node)
+ except FormatOn as f_on:
+ f_on.trim_prefix(node)
+ yield from self.line()
+ yield from self.visit(node)
+
def __attrs_post_init__(self) -> None:
"""You are in a twisty little maze of passages."""
v = self.visit_stmt
return SPACE if prevp.type == token.COMMA else NO
if prevp.type == token.EQUAL:
- if prevp.parent and prevp.parent.type in {
- syms.arglist,
- syms.argument,
- syms.parameters,
- syms.typedargslist,
- syms.varargslist,
- }:
- return NO
+ if prevp.parent:
+ if prevp.parent.type in {
+ syms.arglist, syms.argument, syms.parameters, syms.varargslist
+ }:
+ return NO
+
+ elif prevp.parent.type == syms.typedargslist:
+ # A bit hacky: if the equal sign has whitespace, it means we
+ # previously found it's a typed argument. So, we're using
+ # that, too.
+ return prevp.prefix
elif prevp.type == token.DOUBLESTAR:
if prevp.parent and prevp.parent.type in {
):
return NO
+ elif (
+ prevp.type == token.RIGHTSHIFT
+ and prevp.parent
+ and prevp.parent.type == syms.shift_expr
+ and prevp.prev_sibling
+ and prevp.prev_sibling.type == token.NAME
+ and prevp.prev_sibling.value == 'print' # type: ignore
+ ):
+ # Python 2 print chevron
+ return NO
+
elif prev.type in OPENING_BRACKETS:
return NO
if not prev or prev.type != token.COMMA:
return NO
- if p.type == syms.varargslist:
+ elif p.type == syms.varargslist:
# lambdas
if t == token.RPAR:
return NO
if '#' not in p:
return
+ consumed = 0
nlines = 0
for index, line in enumerate(p.split('\n')):
+ consumed += len(line) + 1 # adding the length of the split '\n'
line = line.lstrip()
if not line:
nlines += 1
comment_type = token.COMMENT # simple trailing comment
else:
comment_type = STANDALONE_COMMENT
- yield Leaf(comment_type, make_comment(line), prefix='\n' * nlines)
+ comment = make_comment(line)
+ yield Leaf(comment_type, comment, prefix='\n' * nlines)
+
+ if comment in {'# fmt: on', '# yapf: enable'}:
+ raise FormatOn(consumed)
+
+ if comment in {'# fmt: off', '# yapf: disable'}:
+ raise FormatOff(consumed)
nlines = 0
if content[0] == '#':
content = content[1:]
- if content and content[0] not in {' ', '!', '#'}:
+ if content and content[0] not in ' !:#':
content = ' ' + content
return '#' + content
If `py36` is True, splitting may generate syntax that is only compatible
with Python 3.6 and later.
"""
+ if isinstance(line, UnformattedLines):
+ yield line
+ return
+
line_str = str(line).strip('\n')
if len(line_str) <= line_length and '\n' not in line_str:
yield line
try:
src_ast = ast.parse(src)
except Exception as exc:
- raise AssertionError(f"cannot parse source: {exc}") from None
+ major, minor = sys.version_info[:2]
+ raise AssertionError(
+ f"cannot use --safe with this file; failed to parse source file "
+ f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
+ f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
+ )
try:
dst_ast = ast.parse(dst)
projects / etc / vim.git / blobdiff