]> git.madduck.net Git - etc/vim.git/blob - fuzz.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

don't require typed-ast
[etc/vim.git] / fuzz.py
1 """Property-based tests for Black.
2
3 By Zac Hatfield-Dodds, based on my Hypothesmith tool for source code
4 generation.  You can run this file with `python`, `pytest`, or (soon)
5 a coverage-guided fuzzer I'm working on.
6 """
7
8 import re
9
10 import hypothesmith
11 from hypothesis import HealthCheck, given, settings, strategies as st
12
13 import black
14 from blib2to3.pgen2.tokenize import TokenError
15
16
17 # This test uses the Hypothesis and Hypothesmith libraries to generate random
18 # syntatically-valid Python source code and run Black in odd modes.
19 @settings(
20     max_examples=1000,  # roughly 1k tests/minute, or half that under coverage
21     derandomize=True,  # deterministic mode to avoid CI flakiness
22     deadline=None,  # ignore Hypothesis' health checks; we already know that
23     suppress_health_check=HealthCheck.all(),  # this is slow and filter-heavy.
24 )
25 @given(
26     # Note that while Hypothesmith might generate code unlike that written by
27     # humans, it's a general test that should pass for any *valid* source code.
28     # (so e.g. running it against code scraped of the internet might also help)
29     src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
30     # Using randomly-varied modes helps us to exercise less common code paths.
31     mode=st.builds(
32         black.FileMode,
33         line_length=st.just(88) | st.integers(0, 200),
34         string_normalization=st.booleans(),
35         is_pyi=st.booleans(),
36     ),
37 )
38 def test_idempotent_any_syntatically_valid_python(
39     src_contents: str, mode: black.FileMode
40 ) -> None:
41     # Before starting, let's confirm that the input string is valid Python:
42     compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith
43
44     # Then format the code...
45     try:
46         dst_contents = black.format_str(src_contents, mode=mode)
47     except black.InvalidInput:
48         # This is a bug - if it's valid Python code, as above, Black should be
49         # able to cope with it.  See issues #970, #1012, #1358, and #1557.
50         # TODO: remove this try-except block when issues are resolved.
51         return
52     except TokenError as e:
53         if (  # Special-case logic for backslashes followed by newlines or end-of-input
54             e.args[0] == "EOF in multi-line statement"
55             and re.search(r"\\($|\r?\n)", src_contents) is not None
56         ):
57             # This is a bug - if it's valid Python code, as above, Black should be
58             # able to cope with it.  See issue #1012.
59             # TODO: remove this block when the issue is resolved.
60             return
61         raise
62
63     # And check that we got equivalent and stable output.
64     black.assert_equivalent(src_contents, dst_contents)
65     black.assert_stable(src_contents, dst_contents, mode=mode)
66
67     # Future test: check that pure-python and mypyc versions of black
68     # give identical output for identical input?
69
70
71 if __name__ == "__main__":
72     # Run tests, including shrinking and reporting any known failures.
73     test_idempotent_any_syntatically_valid_python()
74
75     # If Atheris is available, run coverage-guided fuzzing.
76     # (if you want only bounded fuzzing, just use `pytest fuzz.py`)
77     try:
78         import sys
79         import atheris
80     except ImportError:
81         pass
82     else:
83         test = test_idempotent_any_syntatically_valid_python
84         atheris.Setup(sys.argv, test.hypothesis.fuzz_one_input)
85         atheris.Fuzz()