From: Zac-HD Date: Fri, 21 Aug 2020 10:58:58 +0000 (+1000) Subject: Property-based fuzz test X-Git-Url: https://git.madduck.net/etc/vim.git/commitdiff_plain/cd3a93a14689f046468ece2a5b1f78863c3c4cd2 Property-based fuzz test --- diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..92caa0f --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,31 @@ +name: Fuzz + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade coverage + python -m pip install --upgrade hypothesmith + python -m pip install -e ".[d]" + + - name: Run fuzz tests + run: | + coverage run fuzz.py + coverage report diff --git a/.gitignore b/.gitignore index 509797e..6b94cac 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ src/_black_version.py .eggs .dmypy.json *.swp +.hypothesis/ diff --git a/README.md b/README.md index 44f2d20..20f6fa4 100644 --- a/README.md +++ b/README.md @@ -684,3 +684,4 @@ Multiple contributions by: - Yazdan - [Yngve Høiseth](mailto:yngve@hoiseth.net) - [Yurii Karabas](mailto:1998uriyyo@gmail.com) +- [Zac Hatfield-Dodds](mailto:zac@zhd.dev) diff --git a/fuzz.py b/fuzz.py new file mode 100644 index 0000000..fdd4917 --- /dev/null +++ b/fuzz.py @@ -0,0 +1,59 @@ +"""Property-based tests for Black. + +By Zac Hatfield-Dodds, based on my Hypothesmith tool for source code +generation. You can run this file with `python`, `pytest`, or (soon) +a coverage-guided fuzzer I'm working on. +""" + +import hypothesmith +from hypothesis import HealthCheck, given, settings, strategies as st + +import black + + +# This test uses the Hypothesis and Hypothesmith libraries to generate random +# syntatically-valid Python source code and run Black in odd modes. +@settings( + max_examples=1000, # roughly 1k tests/minute, or half that under coverage + derandomize=True, # deterministic mode to avoid CI flakiness + deadline=None, # ignore Hypothesis' health checks; we already know that + suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy. +) +@given( + # Note that while Hypothesmith might generate code unlike that written by + # humans, it's a general test that should pass for any *valid* source code. + # (so e.g. running it against code scraped of the internet might also help) + src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(), + # Using randomly-varied modes helps us to exercise less common code paths. + mode=st.builds( + black.FileMode, + line_length=st.just(88) | st.integers(0, 200), + string_normalization=st.booleans(), + is_pyi=st.booleans(), + ), +) +def test_idempotent_any_syntatically_valid_python( + src_contents: str, mode: black.FileMode +) -> None: + # Before starting, let's confirm that the input string is valid Python: + compile(src_contents, "", "exec") # else the bug is in hypothesmith + + # Then format the code... + try: + dst_contents = black.format_str(src_contents, mode=mode) + except black.InvalidInput: + # This is a bug - if it's valid Python code, as above, black should be + # able to code with it. See issues #970, #1012, #1358, and #1557. + # TODO: remove this try-except block when issues are resolved. + return + + # And check that we got equivalent and stable output. + black.assert_equivalent(src_contents, dst_contents) + black.assert_stable(src_contents, dst_contents, mode=mode) + + # Future test: check that pure-python and mypyc versions of black + # give identical output for identical input? + + +if __name__ == "__main__": + test_idempotent_any_syntatically_valid_python()