]> git.madduck.net Git - etc/vim.git/blob - fuzz.py

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

6330c58cc810f11b413c8762812806fd0754a970
[etc/vim.git] / fuzz.py
1 """Property-based tests for Black.
2
3 By Zac Hatfield-Dodds, based on my Hypothesmith tool for source code
4 generation.  You can run this file with `python`, `pytest`, or (soon)
5 a coverage-guided fuzzer I'm working on.
6 """
7
8 import hypothesmith
9 from hypothesis import HealthCheck, given, settings, strategies as st
10
11 import black
12
13
14 # This test uses the Hypothesis and Hypothesmith libraries to generate random
15 # syntatically-valid Python source code and run Black in odd modes.
16 @settings(
17     max_examples=1000,  # roughly 1k tests/minute, or half that under coverage
18     derandomize=True,  # deterministic mode to avoid CI flakiness
19     deadline=None,  # ignore Hypothesis' health checks; we already know that
20     suppress_health_check=HealthCheck.all(),  # this is slow and filter-heavy.
21 )
22 @given(
23     # Note that while Hypothesmith might generate code unlike that written by
24     # humans, it's a general test that should pass for any *valid* source code.
25     # (so e.g. running it against code scraped of the internet might also help)
26     src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
27     # Using randomly-varied modes helps us to exercise less common code paths.
28     mode=st.builds(
29         black.FileMode,
30         line_length=st.just(88) | st.integers(0, 200),
31         string_normalization=st.booleans(),
32         is_pyi=st.booleans(),
33     ),
34 )
35 def test_idempotent_any_syntatically_valid_python(
36     src_contents: str, mode: black.FileMode
37 ) -> None:
38     # Before starting, let's confirm that the input string is valid Python:
39     compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith
40
41     # Then format the code...
42     try:
43         dst_contents = black.format_str(src_contents, mode=mode)
44     except black.InvalidInput:
45         # This is a bug - if it's valid Python code, as above, Black should be
46         # able to cope with it.  See issues #970, #1012, #1358, and #1557.
47         # TODO: remove this try-except block when issues are resolved.
48         return
49
50     # And check that we got equivalent and stable output.
51     black.assert_equivalent(src_contents, dst_contents)
52     black.assert_stable(src_contents, dst_contents, mode=mode)
53
54     # Future test: check that pure-python and mypyc versions of black
55     # give identical output for identical input?
56
57
58 if __name__ == "__main__":
59     # Run tests, including shrinking and reporting any known failures.
60     test_idempotent_any_syntatically_valid_python()
61
62     # If Atheris is available, run coverage-guided fuzzing.
63     # (if you want only bounded fuzzing, just use `pytest fuzz.py`)
64     try:
65         import sys
66         import atheris
67     except ImportError:
68         pass
69     else:
70         test = test_idempotent_any_syntatically_valid_python
71         atheris.Setup(sys.argv, test.hypothesis.fuzz_one_input)
72         atheris.Fuzz()