All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 Parse Python code and perform AST validation.
7 from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union
9 if sys.version_info < (3, 8):
10 from typing_extensions import Final
12 from typing import Final
14 from black.mode import Feature, TargetVersion, supports_feature
15 from black.nodes import syms
16 from blib2to3 import pygram
17 from blib2to3.pgen2 import driver
18 from blib2to3.pgen2.grammar import Grammar
19 from blib2to3.pgen2.parse import ParseError
20 from blib2to3.pgen2.tokenize import TokenError
21 from blib2to3.pytree import Leaf, Node
25 _IS_PYPY = platform.python_implementation() == "PyPy"
28 from typed_ast import ast3
30 if sys.version_info < (3, 8) and not _IS_PYPY:
32 "The typed_ast package is required but not installed.\n"
33 "You can upgrade to Python 3.8+ or install typed_ast with\n"
34 "`python3 -m pip install typed-ast`.",
42 PY2_HINT: Final = "Python 2 support was removed in version 22.0."
45 class InvalidInput(ValueError):
46 """Raised when input source code fails all parse attempts."""
49 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
50 if not target_versions:
51 # No target_version specified, so try all grammars.
54 pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
56 pygram.python_grammar_no_print_statement_no_exec_statement,
58 pygram.python_grammar_soft_keywords,
62 # If we have to parse both, try to parse async as a keyword first
63 if not supports_feature(
64 target_versions, Feature.ASYNC_IDENTIFIERS
65 ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
68 pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
70 if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
72 grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
73 if supports_feature(target_versions, Feature.PATTERN_MATCHING):
75 grammars.append(pygram.python_grammar_soft_keywords)
77 # At least one of the above branches must have been taken, because every Python
78 # version has exactly one of the two 'ASYNC_*' flags
82 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
83 """Given a string with source, return the lib2to3 Node."""
84 if not src_txt.endswith("\n"):
87 grammars = get_grammars(set(target_versions))
89 for grammar in grammars:
90 drv = driver.Driver(grammar)
92 result = drv.parse_string(src_txt, True)
95 except ParseError as pe:
96 lineno, column = pe.context[1]
97 lines = src_txt.splitlines()
99 faulty_line = lines[lineno - 1]
101 faulty_line = "<line number missing in source>"
102 errors[grammar.version] = InvalidInput(
103 f"Cannot parse: {lineno}:{column}: {faulty_line}"
106 except TokenError as te:
107 # In edge cases these are raised; and typically don't have a "faulty_line".
108 lineno, column = te.args[1]
109 errors[grammar.version] = InvalidInput(
110 f"Cannot parse: {lineno}:{column}: {te.args[0]}"
114 # Choose the latest version when raising the actual parsing error.
115 assert len(errors) >= 1
116 exc = errors[max(errors)]
118 if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
119 src_txt, pygram.python_grammar_no_print_statement
121 original_msg = exc.args[0]
122 msg = f"{original_msg}\n{PY2_HINT}"
123 raise InvalidInput(msg) from None
127 if isinstance(result, Leaf):
128 result = Node(syms.file_input, [result])
132 def matches_grammar(src_txt: str, grammar: Grammar) -> bool:
133 drv = driver.Driver(grammar)
135 drv.parse_string(src_txt, True)
136 except (ParseError, TokenError, IndentationError):
142 def lib2to3_unparse(node: Node) -> str:
143 """Given a lib2to3 node, return its string representation."""
148 def parse_single_version(
149 src: str, version: Tuple[int, int]
150 ) -> Union[ast.AST, ast3.AST]:
151 filename = "<unknown>"
152 # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
153 if sys.version_info >= (3, 8) and version >= (3,):
154 return ast.parse(src, filename, feature_version=version, type_comments=True)
157 # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
158 # not much we can do as typed-ast won't work either.
159 if sys.version_info >= (3, 8):
160 return ast3.parse(src, filename, type_comments=True)
162 return ast3.parse(src, filename)
164 # Typed-ast is guaranteed to be used here and automatically tracks type
165 # comments separately.
166 return ast3.parse(src, filename, feature_version=version[1])
168 raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
171 def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
172 # TODO: support Python 4+ ;)
173 versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]
176 for version in sorted(versions, reverse=True):
178 return parse_single_version(src, version)
179 except SyntaxError as e:
183 raise SyntaxError(first_error)
186 ast3_AST: Final[Type[ast3.AST]] = ast3.AST
189 def _normalize(lineend: str, value: str) -> str:
190 # To normalize, we strip any leading and trailing space from
192 stripped: List[str] = [i.strip() for i in value.splitlines()]
193 normalized = lineend.join(stripped)
194 # ...and remove any blank lines at the beginning and end of
196 return normalized.strip()
199 def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]:
200 """Simple visitor generating strings to compare ASTs by content."""
202 node = fixup_ast_constants(node)
204 yield f"{' ' * depth}{node.__class__.__name__}("
206 type_ignore_classes: Tuple[Type[Any], ...]
207 for field in sorted(node._fields): # noqa: F402
208 # TypeIgnore will not be present using pypy < 3.8, so need for this
209 if not (_IS_PYPY and sys.version_info < (3, 8)):
210 # TypeIgnore has only one field 'lineno' which breaks this comparison
211 type_ignore_classes = (ast3.TypeIgnore,)
212 if sys.version_info >= (3, 8):
213 type_ignore_classes += (ast.TypeIgnore,)
214 if isinstance(node, type_ignore_classes):
218 value: object = getattr(node, field)
219 except AttributeError:
222 yield f"{' ' * (depth+1)}{field}="
224 if isinstance(value, list):
226 # Ignore nested tuples within del statements, because we may insert
227 # parentheses and they change the AST.
230 and isinstance(node, (ast.Delete, ast3.Delete))
231 and isinstance(item, (ast.Tuple, ast3.Tuple))
233 for elt in item.elts:
234 yield from stringify_ast(elt, depth + 2)
236 elif isinstance(item, (ast.AST, ast3.AST)):
237 yield from stringify_ast(item, depth + 2)
239 # Note that we are referencing the typed-ast ASTs via global variables and not
240 # direct module attribute accesses because that breaks mypyc. It's probably
241 # something to do with the ast3 variables being marked as Any leading
242 # mypy to think this branch is always taken, leaving the rest of the code
243 # unanalyzed. Tighting up the types for the typed-ast AST types avoids the
245 elif isinstance(value, (ast.AST, ast3_AST)):
246 yield from stringify_ast(value, depth + 2)
250 # Constant strings may be indented across newlines, if they are
251 # docstrings; fold spaces after newlines when comparing. Similarly,
252 # trailing and leading space may be removed.
254 isinstance(node, ast.Constant)
256 and isinstance(value, str)
258 normalized = _normalize("\n", value)
261 yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}"
263 yield f"{' ' * depth}) # /{node.__class__.__name__}"
266 def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]:
267 """Map ast nodes deprecated in 3.8 to Constant."""
268 if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)):
269 return ast.Constant(value=node.s)
271 if isinstance(node, (ast.Num, ast3.Num)):
272 return ast.Constant(value=node.n)
274 if isinstance(node, (ast.NameConstant, ast3.NameConstant)):
275 return ast.Constant(value=node.value)