]> git.madduck.net Git - etc/vim.git/blob - src/black/parsing.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Clarify check argument is needed for github action workflow (#3325)
[etc/vim.git] / src / black / parsing.py
1 """
2 Parse Python code and perform AST validation.
3 """
4 import ast
5 import platform
6 import sys
7 from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union
8
9 if sys.version_info < (3, 8):
10     from typing_extensions import Final
11 else:
12     from typing import Final
13
14 from black.mode import Feature, TargetVersion, supports_feature
15 from black.nodes import syms
16 from blib2to3 import pygram
17 from blib2to3.pgen2 import driver
18 from blib2to3.pgen2.grammar import Grammar
19 from blib2to3.pgen2.parse import ParseError
20 from blib2to3.pgen2.tokenize import TokenError
21 from blib2to3.pytree import Leaf, Node
22
23 ast3: Any
24
25 _IS_PYPY = platform.python_implementation() == "PyPy"
26
27 try:
28     from typed_ast import ast3
29 except ImportError:
30     if sys.version_info < (3, 8) and not _IS_PYPY:
31         print(
32             "The typed_ast package is required but not installed.\n"
33             "You can upgrade to Python 3.8+ or install typed_ast with\n"
34             "`python3 -m pip install typed-ast`.",
35             file=sys.stderr,
36         )
37         sys.exit(1)
38     else:
39         ast3 = ast
40
41
42 PY2_HINT: Final = "Python 2 support was removed in version 22.0."
43
44
45 class InvalidInput(ValueError):
46     """Raised when input source code fails all parse attempts."""
47
48
49 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
50     if not target_versions:
51         # No target_version specified, so try all grammars.
52         return [
53             # Python 3.7+
54             pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
55             # Python 3.0-3.6
56             pygram.python_grammar_no_print_statement_no_exec_statement,
57             # Python 3.10+
58             pygram.python_grammar_soft_keywords,
59         ]
60
61     grammars = []
62     # If we have to parse both, try to parse async as a keyword first
63     if not supports_feature(
64         target_versions, Feature.ASYNC_IDENTIFIERS
65     ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
66         # Python 3.7-3.9
67         grammars.append(
68             pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
69         )
70     if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
71         # Python 3.0-3.6
72         grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
73     if supports_feature(target_versions, Feature.PATTERN_MATCHING):
74         # Python 3.10+
75         grammars.append(pygram.python_grammar_soft_keywords)
76
77     # At least one of the above branches must have been taken, because every Python
78     # version has exactly one of the two 'ASYNC_*' flags
79     return grammars
80
81
82 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
83     """Given a string with source, return the lib2to3 Node."""
84     if not src_txt.endswith("\n"):
85         src_txt += "\n"
86
87     grammars = get_grammars(set(target_versions))
88     errors = {}
89     for grammar in grammars:
90         drv = driver.Driver(grammar)
91         try:
92             result = drv.parse_string(src_txt, True)
93             break
94
95         except ParseError as pe:
96             lineno, column = pe.context[1]
97             lines = src_txt.splitlines()
98             try:
99                 faulty_line = lines[lineno - 1]
100             except IndexError:
101                 faulty_line = "<line number missing in source>"
102             errors[grammar.version] = InvalidInput(
103                 f"Cannot parse: {lineno}:{column}: {faulty_line}"
104             )
105
106         except TokenError as te:
107             # In edge cases these are raised; and typically don't have a "faulty_line".
108             lineno, column = te.args[1]
109             errors[grammar.version] = InvalidInput(
110                 f"Cannot parse: {lineno}:{column}: {te.args[0]}"
111             )
112
113     else:
114         # Choose the latest version when raising the actual parsing error.
115         assert len(errors) >= 1
116         exc = errors[max(errors)]
117
118         if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
119             src_txt, pygram.python_grammar_no_print_statement
120         ):
121             original_msg = exc.args[0]
122             msg = f"{original_msg}\n{PY2_HINT}"
123             raise InvalidInput(msg) from None
124
125         raise exc from None
126
127     if isinstance(result, Leaf):
128         result = Node(syms.file_input, [result])
129     return result
130
131
132 def matches_grammar(src_txt: str, grammar: Grammar) -> bool:
133     drv = driver.Driver(grammar)
134     try:
135         drv.parse_string(src_txt, True)
136     except (ParseError, TokenError, IndentationError):
137         return False
138     else:
139         return True
140
141
142 def lib2to3_unparse(node: Node) -> str:
143     """Given a lib2to3 node, return its string representation."""
144     code = str(node)
145     return code
146
147
148 def parse_single_version(
149     src: str, version: Tuple[int, int]
150 ) -> Union[ast.AST, ast3.AST]:
151     filename = "<unknown>"
152     # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
153     if sys.version_info >= (3, 8) and version >= (3,):
154         return ast.parse(src, filename, feature_version=version, type_comments=True)
155
156     if _IS_PYPY:
157         # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
158         # not much we can do as typed-ast won't work either.
159         if sys.version_info >= (3, 8):
160             return ast3.parse(src, filename, type_comments=True)
161         else:
162             return ast3.parse(src, filename)
163     else:
164         # Typed-ast is guaranteed to be used here and automatically tracks type
165         # comments separately.
166         return ast3.parse(src, filename, feature_version=version[1])
167
168
169 def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
170     # TODO: support Python 4+ ;)
171     versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]
172
173     first_error = ""
174     for version in sorted(versions, reverse=True):
175         try:
176             return parse_single_version(src, version)
177         except SyntaxError as e:
178             if not first_error:
179                 first_error = str(e)
180
181     raise SyntaxError(first_error)
182
183
184 ast3_AST: Final[Type[ast3.AST]] = ast3.AST
185
186
187 def _normalize(lineend: str, value: str) -> str:
188     # To normalize, we strip any leading and trailing space from
189     # each line...
190     stripped: List[str] = [i.strip() for i in value.splitlines()]
191     normalized = lineend.join(stripped)
192     # ...and remove any blank lines at the beginning and end of
193     # the whole string
194     return normalized.strip()
195
196
197 def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]:
198     """Simple visitor generating strings to compare ASTs by content."""
199
200     node = fixup_ast_constants(node)
201
202     yield f"{'  ' * depth}{node.__class__.__name__}("
203
204     type_ignore_classes: Tuple[Type[Any], ...]
205     for field in sorted(node._fields):  # noqa: F402
206         # TypeIgnore will not be present using pypy < 3.8, so need for this
207         if not (_IS_PYPY and sys.version_info < (3, 8)):
208             # TypeIgnore has only one field 'lineno' which breaks this comparison
209             type_ignore_classes = (ast3.TypeIgnore,)
210             if sys.version_info >= (3, 8):
211                 type_ignore_classes += (ast.TypeIgnore,)
212             if isinstance(node, type_ignore_classes):
213                 break
214
215         try:
216             value: object = getattr(node, field)
217         except AttributeError:
218             continue
219
220         yield f"{'  ' * (depth+1)}{field}="
221
222         if isinstance(value, list):
223             for item in value:
224                 # Ignore nested tuples within del statements, because we may insert
225                 # parentheses and they change the AST.
226                 if (
227                     field == "targets"
228                     and isinstance(node, (ast.Delete, ast3.Delete))
229                     and isinstance(item, (ast.Tuple, ast3.Tuple))
230                 ):
231                     for elt in item.elts:
232                         yield from stringify_ast(elt, depth + 2)
233
234                 elif isinstance(item, (ast.AST, ast3.AST)):
235                     yield from stringify_ast(item, depth + 2)
236
237         # Note that we are referencing the typed-ast ASTs via global variables and not
238         # direct module attribute accesses because that breaks mypyc. It's probably
239         # something to do with the ast3 variables being marked as Any leading
240         # mypy to think this branch is always taken, leaving the rest of the code
241         # unanalyzed. Tighting up the types for the typed-ast AST types avoids the
242         # mypyc crash.
243         elif isinstance(value, (ast.AST, ast3_AST)):
244             yield from stringify_ast(value, depth + 2)
245
246         else:
247             normalized: object
248             # Constant strings may be indented across newlines, if they are
249             # docstrings; fold spaces after newlines when comparing. Similarly,
250             # trailing and leading space may be removed.
251             if (
252                 isinstance(node, ast.Constant)
253                 and field == "value"
254                 and isinstance(value, str)
255             ):
256                 normalized = _normalize("\n", value)
257             else:
258                 normalized = value
259             yield f"{'  ' * (depth+2)}{normalized!r},  # {value.__class__.__name__}"
260
261     yield f"{'  ' * depth})  # /{node.__class__.__name__}"
262
263
264 def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]:
265     """Map ast nodes deprecated in 3.8 to Constant."""
266     if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)):
267         return ast.Constant(value=node.s)
268
269     if isinstance(node, (ast.Num, ast3.Num)):
270         return ast.Constant(value=node.n)
271
272     if isinstance(node, (ast.NameConstant, ast3.NameConstant)):
273         return ast.Constant(value=node.value)
274
275     return node