from blib2to3.pgen2 import driver, token
from blib2to3.pgen2.grammar import Grammar
from blib2to3.pgen2.parse import ParseError
-from blib2to3.pgen2.tokenize import TokenizerConfig
__version__ = "19.3b0"
TRAILING_COMMA_IN_DEF = 5
# The following two feature-flags are mutually exclusive, and exactly one should be
# set for every version of python.
- ASYNC_IS_VALID_IDENTIFIER = 6
- ASYNC_IS_RESERVED_KEYWORD = 7
+ ASYNC_IDENTIFIERS = 6
+ ASYNC_KEYWORDS = 7
VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
- TargetVersion.PY27: {Feature.ASYNC_IS_VALID_IDENTIFIER},
- TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IS_VALID_IDENTIFIER},
- TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IS_VALID_IDENTIFIER},
+ TargetVersion.PY27: {Feature.ASYNC_IDENTIFIERS},
+ TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
+ TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
TargetVersion.PY35: {
Feature.UNICODE_LITERALS,
Feature.TRAILING_COMMA_IN_CALL,
- Feature.ASYNC_IS_VALID_IDENTIFIER,
+ Feature.ASYNC_IDENTIFIERS,
},
TargetVersion.PY36: {
Feature.UNICODE_LITERALS,
Feature.NUMERIC_UNDERSCORES,
Feature.TRAILING_COMMA_IN_CALL,
Feature.TRAILING_COMMA_IN_DEF,
- Feature.ASYNC_IS_VALID_IDENTIFIER,
+ Feature.ASYNC_IDENTIFIERS,
},
TargetVersion.PY37: {
Feature.UNICODE_LITERALS,
Feature.NUMERIC_UNDERSCORES,
Feature.TRAILING_COMMA_IN_CALL,
Feature.TRAILING_COMMA_IN_DEF,
- Feature.ASYNC_IS_RESERVED_KEYWORD,
+ Feature.ASYNC_KEYWORDS,
},
TargetVersion.PY38: {
Feature.UNICODE_LITERALS,
Feature.NUMERIC_UNDERSCORES,
Feature.TRAILING_COMMA_IN_CALL,
Feature.TRAILING_COMMA_IN_DEF,
- Feature.ASYNC_IS_RESERVED_KEYWORD,
+ Feature.ASYNC_KEYWORDS,
},
}
)
if verbose or not quiet:
- bang = "💥 💔 💥" if report.return_code else "✨ 🍰 ✨"
- out(f"All done! {bang}")
+ out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")
click.secho(str(report), err=True)
ctx.exit(report.return_code)
`line_length` determines how many characters per line are allowed.
"""
src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
- dst_contents = ""
+ dst_contents = []
future_imports = get_future_imports(src_node)
if mode.target_versions:
versions = mode.target_versions
}
for current_line in lines.visit(src_node):
for _ in range(after):
- dst_contents += str(empty_line)
+ dst_contents.append(str(empty_line))
before, after = elt.maybe_empty_lines(current_line)
for _ in range(before):
- dst_contents += str(empty_line)
+ dst_contents.append(str(empty_line))
for line in split_line(
current_line, line_length=mode.line_length, features=split_line_features
):
- dst_contents += str(line)
- return dst_contents
+ dst_contents.append(str(line))
+ return "".join(dst_contents)
def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
return tiow.read(), encoding, newline
-@dataclass(frozen=True)
-class ParserConfig:
- grammar: Grammar
- tokenizer_config: TokenizerConfig = TokenizerConfig()
-
-
-def get_parser_configs(target_versions: Set[TargetVersion]) -> List[ParserConfig]:
+def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
if not target_versions:
# No target_version specified, so try all grammars.
return [
# Python 3.7+
- ParserConfig(
- pygram.python_grammar_no_print_statement_no_exec_statement,
- TokenizerConfig(async_is_reserved_keyword=True),
- ),
+ pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
# Python 3.0-3.6
- ParserConfig(
- pygram.python_grammar_no_print_statement_no_exec_statement,
- TokenizerConfig(async_is_reserved_keyword=False),
- ),
+ pygram.python_grammar_no_print_statement_no_exec_statement,
# Python 2.7 with future print_function import
- ParserConfig(pygram.python_grammar_no_print_statement),
+ pygram.python_grammar_no_print_statement,
# Python 2.7
- ParserConfig(pygram.python_grammar),
+ pygram.python_grammar,
]
elif all(version.is_python2() for version in target_versions):
# Python 2-only code, so try Python 2 grammars.
return [
# Python 2.7 with future print_function import
- ParserConfig(pygram.python_grammar_no_print_statement),
+ pygram.python_grammar_no_print_statement,
# Python 2.7
- ParserConfig(pygram.python_grammar),
+ pygram.python_grammar,
]
else:
# Python 3-compatible code, so only try Python 3 grammar.
- configs = []
+ grammars = []
# If we have to parse both, try to parse async as a keyword first
- if not supports_feature(target_versions, Feature.ASYNC_IS_VALID_IDENTIFIER):
+ if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
# Python 3.7+
- configs.append(
- ParserConfig(
- pygram.python_grammar_no_print_statement_no_exec_statement,
- TokenizerConfig(async_is_reserved_keyword=True),
- )
+ grammars.append(
+ pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords # noqa: B950
)
- if not supports_feature(target_versions, Feature.ASYNC_IS_RESERVED_KEYWORD):
+ if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
# Python 3.0-3.6
- configs.append(
- ParserConfig(
- pygram.python_grammar_no_print_statement_no_exec_statement,
- TokenizerConfig(async_is_reserved_keyword=False),
- )
- )
+ grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
# At least one of the above branches must have been taken, because every Python
- # version has exactly one of the two 'ASYNC_IS_*' flags
- return configs
+ # version has exactly one of the two 'ASYNC_*' flags
+ return grammars
def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
if src_txt[-1:] != "\n":
src_txt += "\n"
- for parser_config in get_parser_configs(set(target_versions)):
- drv = driver.Driver(
- parser_config.grammar,
- pytree.convert,
- tokenizer_config=parser_config.tokenizer_config,
- )
+ for grammar in get_grammars(set(target_versions)):
+ drv = driver.Driver(grammar, pytree.convert)
try:
result = drv.parse_string(src_txt, True)
break
self.current_line.append(node)
yield from super().visit_default(node)
+ def visit_atom(self, node: Node) -> Iterator[Line]:
+ # Always make parentheses invisible around a single node, because it should
+ # not be needed (except in the case of yield, where removing the parentheses
+ # produces a SyntaxError).
+ if (
+ len(node.children) == 3
+ and isinstance(node.children[0], Leaf)
+ and node.children[0].type == token.LPAR
+ and isinstance(node.children[2], Leaf)
+ and node.children[2].type == token.RPAR
+ and isinstance(node.children[1], Leaf)
+ and not (
+ node.children[1].type == token.NAME
+ and node.children[1].value == "yield"
+ )
+ ):
+ node.children[0].value = ""
+ node.children[2].value = ""
+ yield from super().visit_default(node)
+
def visit_INDENT(self, node: Node) -> Iterator[Line]:
"""Increase indentation level, maybe yield a line."""
# In blib2to3 INDENT never holds comments.
new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
if "f" in prefix.casefold():
- matches = re.findall(r"[^{]\{(.*?)\}[^}]", new_body)
+ matches = re.findall(
+ r"""
+ (?:[^{]|^)\{ # start of the string or a non-{ followed by a single {
+ ([^{].*?) # contents of the brackets except if begins with {{
+ \}(?:[^}]|$) # A } followed by end of the string or a non-}
+ """,
+ new_body,
+ re.VERBOSE,
+ )
for m in matches:
if "\\" in str(m):
# Do not introduce backslashes in interpolated expressions
projects / etc / vim.git / blobdiff