Fix wheel build (#186)

* Bump cibuildwheel==2.19.2 to get manylinux2014 working * Fix spt3g build to never link to python. Compiled python extensions distributed in wheels should never link directly to the core Python libraries, since that would attempt to bundle the python libs on the build system into the wheel, rather than resolving the linking at run time after the wheel is installed. We already patch the spt3g cmake files to not search for python development libraries. This commit updates the patch to also remove building of compiled unit tests and examples, which would require linking to libpython. * In github workflows, change "docker-compose" to "docker compose" * Pin numpy to less than v2 for now * * Bump versions of vendored boost and openblas * Drop support for python 3.7 * Bump version of bundled spt3g to 4bd3275 * Force numpy requirements to 1.x * Use Intel-based macos runners for building x86-64 wheels. * Vendor OpenBLAS on macos using the new libscipy_openblas wheel * Fix typo on macos dependency install * Correct homebrew gcc version on macos * Remove forced warning flags added to spt3g build * Force macos minimum deployment target to 10.13 * Force macos minimum deployment target to 13.0 * Update upload-artifact action. Replace cancel-workflow action with native github workflow concurrency stanza * Make artifact upload path unique for each matrix execution * Simplify build matrix for wheels. Fix artifact upload. * Fix typo, add simple version check. * Fix artifact zip path to be unique * Temporarily revert boost to 1.80 to see if it impacts the size of bundled spt3g libraries. * Restore boost 1.86 after test * Remove debugging symbols when building wheels, since it causes a 10x increase in library size. --------- Co-authored-by: Matthew Hasselfield <[email protected]>
simonsobs · Aug 26, 2024 · b665511 · b665511
1 parent 34781f0
commit b665511
Show file tree

Hide file tree

Showing 12 changed files with 166 additions and 151 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -11,68 +11,63 @@ env:
   TWINE_USERNAME: __token__
   TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build_wheels:
-    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
+    name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       # Ensure that a wheel builder finishes even if another fails.  Useful for
       # debugging multiple problems in parallel.
       fail-fast: false
       matrix:
         include:
-          # Linux 64 bit manylinux2014
-          - os: ubuntu-latest
-            python: 37
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+          # Linux 64 bit manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 38
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 39
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 310
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 311
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
 
-          # MacOS x86_64
-          - os: macos-latest
+          # MacOS x86_64.  The macos-13 runner is the last
+          # Intel-based runner version.  At some point we'll
+          # need to switch to macos-14 and test cross compiling.
+          - os: macos-13
+            arch: x86_64
             python: 39
-            platform_id: macosx_x86_64
-          - os: macos-latest
+            builder: macosx
+          - os: macos-13
+            arch: x86_64
             python: 310
-            platform_id: macosx_x86_64
-          - os: macos-latest
+            builder: macosx
+          - os: macos-13
+            arch: x86_64
             python: 311
-            platform_id: macosx_x86_64
-
-          # MacOS arm64 - Re-enable after github CI supports M1 runners:
-          #   https://github.com/actions/virtual-environments/issues/2187
-          #
-          # - os: macos-11
-          #   python: 39
-          #   platform_id: macosx_arm64
-          # - os: macos-11
-          #   python: 310
-          #   platform_id: macosx_arm64
+            builder: macosx
     env:
-      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
-      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
-      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
+      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
+      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
+      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
       CIBW_TEST_SKIP: "*-macosx_arm64"
       CIBW_BUILD_VERBOSITY: 3
       CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local"
-      CIBW_ENVIRONMENT_MACOS: "CC=gcc-12 CXX=g++-12 CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local"
+      CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib"
       CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
-      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.platform_id }}
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }}
       CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel}
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs}
       CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2
@@ -81,27 +76,22 @@ jobs:
         python -c 'import so3g.smurf.reader; from spt3g import core' &&
         python -m pytest {package}/test
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ github.token }}
-
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: '3.10'
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install twine cibuildwheel==2.15.0
+          python -m pip install twine cibuildwheel==2.20.0
 
       - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
       - name: Upload to PyPI
         run: |
-          python -m twine upload wheelhouse/so3g*.whl
+          python -m twine upload wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl
diff --git a/.github/workflows/official-docker-images.yml b/.github/workflows/official-docker-images.yml
@@ -24,7 +24,7 @@ jobs:
     # Build
     - name: Build docker images
       run: |
-        docker-compose build
+        docker compose build
 
     # Test (already been run by pytest workflow, but they don't take long...)
     - name: Test with pytest within a docker container
@@ -54,10 +54,10 @@ jobs:
         export DOCKER_TAG=`git describe --tags --always`
 
         # Tag all images for upload to the registry
-        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest
-        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
+        docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest
+        docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
 
         # Upload to docker registry
-        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest
-        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
-        docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed
+        docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest
+        docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG}
+        docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -5,15 +5,15 @@ on:
     branches-ignore: [ master ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ github.token }}
     - name: Checkout
       uses: actions/checkout@v2
     - name: Set up Python 3.8
@@ -23,7 +23,7 @@ jobs:
 
     - name: Build docker images
       run: |
-        docker-compose build
+        docker compose build
 
     - name: Test with pytest within a docker container
       run: |

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -3,69 +3,62 @@ name:  Test Binary Wheels
 
 on: workflow_dispatch
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build_wheels:
-    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
+    name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       # Ensure that a wheel builder finishes even if another fails.  Useful for
       # debugging multiple problems in parallel.
       fail-fast: false
       matrix:
         include:
-          # Linux 64 bit manylinux2014
-          - os: ubuntu-latest
-            python: 37
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+          # Linux 64 bit manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 38
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 39
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 310
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 311
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
 
-          # MacOS x86_64
-          - os: macos-latest
+          # MacOS x86_64.  The macos-13 runner is the last
+          # Intel-based runner version.  At some point we'll
+          # need to switch to macos-14 and test cross compiling.
+          - os: macos-13
+            arch: x86_64
             python: 39
-            platform_id: macosx_x86_64
-          - os: macos-latest
+            builder: macosx
+          - os: macos-13
+            arch: x86_64
             python: 310
-            platform_id: macosx_x86_64
-          - os: macos-latest
+            builder: macosx
+          - os: macos-13
+            arch: x86_64
             python: 311
-            platform_id: macosx_x86_64
-
-          # MacOS arm64 - Re-enable after github CI supports M1 runners:
-          #   https://github.com/actions/virtual-environments/issues/2187
-          #
-          #  Note:  this may also require a revisit of clang / cereal errors.
-          #
-          # - os: macos-11
-          #   python: 39
-          #   platform_id: macosx_arm64
-          # - os: macos-11
-          #   python: 310
-          #   platform_id: macosx_arm64
+            builder: macosx
     env:
-      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
-      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
-      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
+      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
+      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
+      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
       CIBW_BUILD_VERBOSITY: 3
-      CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local"
-      CIBW_ENVIRONMENT_MACOS: "CC=gcc-12 CXX=g++-12 CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local"
+      CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local"
+      CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib"
       CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
-      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.platform_id }}
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }}
       CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel}
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs}
       CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2
@@ -74,28 +67,23 @@ jobs:
         python -c 'import so3g.smurf.reader; from spt3g import core' &&
         python -m pytest {package}/test
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ github.token }}
-
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: '3.10'
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.15.0
+          python -m pip install cibuildwheel==2.20.0
 
       - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: wheels
-          path: ./wheelhouse/so3g*.whl
+          name: wheels_cp${{ matrix.python }}-${{ matrix.builder }}
+          path: ./wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@
 # A containerized so3g installation.
 
 # Build on spt3g base image
-FROM simonsobs/spt3g:0.3-276-gbe879ce
+FROM simonsobs/spt3g:0.3-289-g4bd3275
 
 # Set locale
 ENV LANG C.UTF-8

diff --git a/python/__init__.py b/python/__init__.py
@@ -1,4 +1,12 @@
 import os
+import numpy as np
+
+# Verify that we are using numpy 1.x
+npversion = np.version.version
+npversplit = npversion.split(".")
+if npversplit[0] != "1":
+    msg = f"so3g requires numpy version 1.x but found version {npversion}"
+    raise RuntimeError(msg)
 
 if os.getenv('DOCS_BUILD') == '1':
     from ._libso3g_docstring_shells import *

diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 astropy
 matplotlib
-numpy
+numpy<2
 scipy
 ephem
 pytz

diff --git a/setup.py b/setup.py
@@ -373,7 +373,7 @@ def readme():
 conf["python_requires"] = ">=3.7.0"
 conf["setup_requires"] = (["wheel", "cmake"],)
 conf["install_requires"] = [
-    "numpy",
+    "numpy<2",
     "astropy",
     "matplotlib",
     "scipy",

diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh
@@ -33,7 +33,7 @@ PREFIX=/usr/local
 pip install --upgrade pip
 
 # Install a couple of base packages that are always required
-pip install -v cmake wheel
+pip install -v cmake wheel setuptools
 
 # In order to maximize ABI compatibility with numpy, build with the newest numpy
 # version containing the oldest ABI version compatible with the python we are using.
@@ -59,7 +59,7 @@ CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements
 
 # Install Openblas
 
-openblas_version=0.3.21
+openblas_version=0.3.28
 openblas_dir=OpenBLAS-${openblas_version}
 openblas_pkg=${openblas_dir}.tar.gz
 
@@ -83,14 +83,14 @@ tar xzf ${openblas_pkg} \
 
 # Install boost
 
-boost_version=1_80_0
+boost_version=1_86_0
 boost_dir=boost_${boost_version}
 boost_pkg=${boost_dir}.tar.bz2
 
 echo "Fetching boost..."
 
 if [ ! -e ${boost_pkg} ]; then
-    curl -SL "https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/${boost_pkg}" -o "${boost_pkg}"
+    curl -SL "https://archives.boost.io/release/1.86.0/source/${boost_pkg}" -o "${boost_pkg}"
 fi
 
 echo "Building boost..."
-Original file line number
+Diff line change
@@ -1,6 +1,6 @@
     astropy
     matplotlib
-    numpy
+    numpy<2
     scipy
     ephem
     pytz
@@ Expand Down @@