Port & upstream mypyc wheel build workflow (#3197)

diff --git a/.github/mypyc-requirements.txt b/.github/mypyc-requirements.txt
index 4542673174ca0327857ddf93f8d4c93220baff51..352d36c00708c3018d07e69152b5ab046282f04b 100644 (file)
--- a/.github/mypyc-requirements.txt
+++ b/.github/mypyc-requirements.txt
@@ -1,4 +1,4 @@
-mypy == 0.920
+mypy == 0.971

 
 # A bunch of packages for type information
 mypy-extensions >= 0.4.3
 
 # A bunch of packages for type information
 mypy-extensions >= 0.4.3

diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml
index cda215aa5d62e78ad84fd47629bfb11464f37b8f..31a83266345076a845a9966ced071e236b4778ce 100644 (file)
--- a/.github/workflows/pypi_upload.yml
+++ b/.github/workflows/pypi_upload.yml
@@ -1,4 +1,4 @@
-name: pypi_upload
+name: Publish to PyPI

 
 on:
   release:
 
 on:
   release:


@@ -8,14 +8,14 @@ permissions:
   contents: read
 
 jobs:
   contents: read
 
 jobs:

-  build:
-    name: PyPI Upload
+  main:
+    name: sdist + pure wheel

     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 

-      - name: Set up Python
+      - name: Set up latest Python

         uses: actions/setup-python@v4
         with:
           python-version: "*"
         uses: actions/setup-python@v4
         with:
           python-version: "*"


@@ -26,11 +26,51 @@ jobs:
           python -m pip install --upgrade build twine
 
       - name: Build wheel and source distributions
           python -m pip install --upgrade build twine
 
       - name: Build wheel and source distributions

-        run: |
-          python -m build
+        run: python -m build

 
       - name: Upload to PyPI via Twine
         env:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
 
       - name: Upload to PyPI via Twine
         env:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

-        run: |
-          twine upload --verbose -u '__token__' dist/*
+        run: twine upload --verbose -u '__token__' dist/*
+
+  mypyc:
+    name: mypyc wheels (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            name: linux-x86_64
+          - os: windows-2019
+            name: windows-amd64
+          - os: macos-11
+            name: macos-x86_64
+            macos_arch: "x86_64"
+          - os: macos-11
+            name: macos-arm64
+            macos_arch: "arm64"
+          - os: macos-11
+            name: macos-universal2
+            macos_arch: "universal2"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build wheels via cibuildwheel
+        uses: pypa/cibuildwheel@v2.8.1
+        env:
+          CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}"
+          # This isn't supported in pyproject.toml which makes sense (but is annoying).
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6.2"
+
+      - name: Upload wheels as workflow artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.name }}-mypyc-wheels
+          path: ./wheelhouse/*.whl
+
+      - name: Upload wheels to PyPI via Twine
+        env:
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl

diff --git a/pyproject.toml b/pyproject.toml
index 367650720561267ce7dc6bcaf5c5802130f47d83..813e86b2e930a658dae49b9b406527d7fb4279f8 100644 (file)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,6 +29,58 @@ preview = true
 requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"]
 build-backend = "setuptools.build_meta"
 
 requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"]
 build-backend = "setuptools.build_meta"
 

+[tool.cibuildwheel]
+build-verbosity = 1
+# So these are the environments we target:
+# - Python: CPython 3.6+ only
+# - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64
+# - OS: Linux (no musl), Windows, and macOS
+build = "cp3*-*"
+skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"]
+before-build = ["pip install -r .github/mypyc-requirements.txt"]
+# This is the bare minimum needed to run the test suite. Pulling in the full
+# test_requirements.txt would download a bunch of other packages not necessary
+# here and would slow down the testing step a fair bit.
+test-requires = ["pytest>=6.1.1"]
+test-command = 'pytest {project} -k "not incompatible_with_mypyc"'
+test-extras = ["d"," jupyter"]
+# Skip trying to test arm64 builds on Intel Macs. (so cross-compilation doesn't
+# straight up crash)
+test-skip = ["*-macosx_arm64", "*-macosx_universal2:arm64"]
+
+[tool.cibuildwheel.environment]
+BLACK_USE_MYPYC = "1"
+MYPYC_OPT_LEVEL = "3"
+MYPYC_DEBUG_LEVEL = "0"
+# The dependencies required to build wheels with mypyc aren't specified in
+# [build-system].requires so we'll have to manage the build environment ourselves.
+PIP_NO_BUILD_ISOLATION = "no"
+
+[tool.cibuildwheel.linux]
+before-build = [
+    "pip install -r .github/mypyc-requirements.txt",
+    "yum install -y clang",
+]
+# Newer images break the builds, not sure why. We'll need to investigate more later.
+manylinux-x86_64-image = "quay.io/pypa/manylinux2014_x86_64:2021-11-20-f410d11"
+
+[tool.cibuildwheel.linux.environment]
+BLACK_USE_MYPYC = "1"
+MYPYC_OPT_LEVEL = "3"
+MYPYC_DEBUG_LEVEL = "0"
+PIP_NO_BUILD_ISOLATION = "no"
+# Black needs Clang to compile successfully on Linux.
+CC = "clang"
+
+[tool.cibuildwheel.windows]
+# For some reason, (compiled) mypyc is failing to start up with "ImportError: DLL load
+# failed: A dynamic link library (DLL) initialization routine failed." on Windows for
+# at least 3.6. Let's just use interpreted mypy[c].
+# See also: https://github.com/mypyc/mypyc/issues/819.
+before-build = [
+    "pip install -r .github/mypyc-requirements.txt --no-binary mypy"
+]
+

 [tool.isort]
 atomic = true
 profile = "black"
 [tool.isort]
 atomic = true
 profile = "black"

diff --git a/setup.py b/setup.py
index 3accdf433bc430f3f37e829dc6c2ede514107020..bc0cc32352ec16c613c15e3114a7bdf97e057bfc 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,10 @@ if USE_MYPYC:
     ]
 
     opt_level = os.getenv("MYPYC_OPT_LEVEL", "3")
     ]
 
     opt_level = os.getenv("MYPYC_OPT_LEVEL", "3")

-    ext_modules = mypycify(mypyc_targets, opt_level=opt_level, verbose=True)
+    debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "3")
+    ext_modules = mypycify(
+        mypyc_targets, opt_level=opt_level, debug_level=debug_level, verbose=True
+    )

 else:
     ext_modules = []
 
 else:
     ext_modules = []