]> git.madduck.net Git - etc/vim.git/commitdiff

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

Property-based fuzz test
authorZac-HD <zac.hatfield.dodds@gmail.com>
Fri, 21 Aug 2020 10:58:58 +0000 (20:58 +1000)
committerŁukasz Langa <lukasz@langa.pl>
Fri, 21 Aug 2020 14:47:49 +0000 (16:47 +0200)
.github/workflows/fuzz.yml [new file with mode: 0644]
.gitignore
README.md
fuzz.py [new file with mode: 0644]

diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml
new file mode 100644 (file)
index 0000000..92caa0f
--- /dev/null
@@ -0,0 +1,31 @@
+name: Fuzz
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade coverage
+          python -m pip install --upgrade hypothesmith
+          python -m pip install -e ".[d]"
+
+      - name: Run fuzz tests
+        run: |
+          coverage run fuzz.py
+          coverage report
index 509797e65c4df34f6099182ad57be103b4fa536d..6b94cacd183fd04e182dc6fc3191171107212057 100644 (file)
@@ -14,3 +14,4 @@ src/_black_version.py
 .eggs
 .dmypy.json
 *.swp
 .eggs
 .dmypy.json
 *.swp
+.hypothesis/
index 44f2d207c8b1bca4581384a20049693786fb2fa9..20f6fa420b220778440c65f5b23b3b7d43aee461 100644 (file)
--- a/README.md
+++ b/README.md
@@ -684,3 +684,4 @@ Multiple contributions by:
 - Yazdan
 - [Yngve Høiseth](mailto:yngve@hoiseth.net)
 - [Yurii Karabas](mailto:1998uriyyo@gmail.com)
 - Yazdan
 - [Yngve Høiseth](mailto:yngve@hoiseth.net)
 - [Yurii Karabas](mailto:1998uriyyo@gmail.com)
+- [Zac Hatfield-Dodds](mailto:zac@zhd.dev)
diff --git a/fuzz.py b/fuzz.py
new file mode 100644 (file)
index 0000000..fdd4917
--- /dev/null
+++ b/fuzz.py
@@ -0,0 +1,59 @@
+"""Property-based tests for Black.
+
+By Zac Hatfield-Dodds, based on my Hypothesmith tool for source code
+generation.  You can run this file with `python`, `pytest`, or (soon)
+a coverage-guided fuzzer I'm working on.
+"""
+
+import hypothesmith
+from hypothesis import HealthCheck, given, settings, strategies as st
+
+import black
+
+
+# This test uses the Hypothesis and Hypothesmith libraries to generate random
+# syntatically-valid Python source code and run Black in odd modes.
+@settings(
+    max_examples=1000,  # roughly 1k tests/minute, or half that under coverage
+    derandomize=True,  # deterministic mode to avoid CI flakiness
+    deadline=None,  # ignore Hypothesis' health checks; we already know that
+    suppress_health_check=HealthCheck.all(),  # this is slow and filter-heavy.
+)
+@given(
+    # Note that while Hypothesmith might generate code unlike that written by
+    # humans, it's a general test that should pass for any *valid* source code.
+    # (so e.g. running it against code scraped of the internet might also help)
+    src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
+    # Using randomly-varied modes helps us to exercise less common code paths.
+    mode=st.builds(
+        black.FileMode,
+        line_length=st.just(88) | st.integers(0, 200),
+        string_normalization=st.booleans(),
+        is_pyi=st.booleans(),
+    ),
+)
+def test_idempotent_any_syntatically_valid_python(
+    src_contents: str, mode: black.FileMode
+) -> None:
+    # Before starting, let's confirm that the input string is valid Python:
+    compile(src_contents, "<string>", "exec")  # else the bug is in hypothesmith
+
+    # Then format the code...
+    try:
+        dst_contents = black.format_str(src_contents, mode=mode)
+    except black.InvalidInput:
+        # This is a bug - if it's valid Python code, as above, black should be
+        # able to code with it.  See issues #970, #1012, #1358, and #1557.
+        # TODO: remove this try-except block when issues are resolved.
+        return
+
+    # And check that we got equivalent and stable output.
+    black.assert_equivalent(src_contents, dst_contents)
+    black.assert_stable(src_contents, dst_contents, mode=mode)
+
+    # Future test: check that pure-python and mypyc versions of black
+    # give identical output for identical input?
+
+
+if __name__ == "__main__":
+    test_idempotent_any_syntatically_valid_python()