diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4e97211..293d4da 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,9 +11,13 @@ env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} runs-on: ${{ matrix.os }} strategy: # Ensure that a wheel builder finishes even if another fails. Useful for @@ -21,58 +25,49 @@ jobs: fail-fast: false matrix: include: - # Linux 64 bit manylinux2014 - - os: ubuntu-latest - python: 37 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + # Linux 64 bit manylinux - os: ubuntu-latest + arch: x86_64 python: 38 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 39 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 310 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 311 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - # MacOS x86_64 - - os: macos-latest + # MacOS x86_64. The macos-13 runner is the last + # Intel-based runner version. At some point we'll + # need to switch to macos-14 and test cross compiling. + - os: macos-13 + arch: x86_64 python: 39 - platform_id: macosx_x86_64 - - os: macos-latest + builder: macosx + - os: macos-13 + arch: x86_64 python: 310 - platform_id: macosx_x86_64 - - os: macos-latest + builder: macosx + - os: macos-13 + arch: x86_64 python: 311 - platform_id: macosx_x86_64 - - # MacOS arm64 - Re-enable after github CI supports M1 runners: - # https://github.com/actions/virtual-environments/issues/2187 - # - # - os: macos-11 - # python: 39 - # platform_id: macosx_arm64 - # - os: macos-11 - # python: 310 - # platform_id: macosx_arm64 + builder: macosx env: - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014 CIBW_TEST_SKIP: "*-macosx_arm64" CIBW_BUILD_VERBOSITY: 3 CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local" - CIBW_ENVIRONMENT_MACOS: "CC=gcc-12 CXX=g++-12 CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib" CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh - CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.platform_id }} + CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }} CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel} CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs} CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2 @@ -81,22 +76,17 @@ jobs: python -c 'import so3g.smurf.reader; from spt3g import core' && python -m pytest {package}/test steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.10' - name: Install cibuildwheel run: | - python -m pip install twine cibuildwheel==2.15.0 + python -m pip install twine cibuildwheel==2.20.0 - name: Build wheel run: | @@ -104,4 +94,4 @@ jobs: - name: Upload to PyPI run: | - python -m twine upload wheelhouse/so3g*.whl + python -m twine upload wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl diff --git a/.github/workflows/official-docker-images.yml b/.github/workflows/official-docker-images.yml index d66a5d7..795bd7c 100644 --- a/.github/workflows/official-docker-images.yml +++ b/.github/workflows/official-docker-images.yml @@ -24,7 +24,7 @@ jobs: # Build - name: Build docker images run: | - docker-compose build + docker compose build # Test (already been run by pytest workflow, but they don't take long...) - name: Test with pytest within a docker container @@ -54,10 +54,10 @@ jobs: export DOCKER_TAG=`git describe --tags --always` # Tag all images for upload to the registry - docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest - docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} + docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest + docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} # Upload to docker registry - docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest - docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} - docker-compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed + docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest + docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} + docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 40f1fda..0864e10 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -5,15 +5,15 @@ on: branches-ignore: [ master ] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - name: Checkout uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -23,7 +23,7 @@ jobs: - name: Build docker images run: | - docker-compose build + docker compose build - name: Test with pytest within a docker container run: | diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 46b48a7..af4d1dc 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -3,9 +3,13 @@ name: Test Binary Wheels on: workflow_dispatch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }} + name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} runs-on: ${{ matrix.os }} strategy: # Ensure that a wheel builder finishes even if another fails. Useful for @@ -13,59 +17,48 @@ jobs: fail-fast: false matrix: include: - # Linux 64 bit manylinux2014 - - os: ubuntu-latest - python: 37 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + # Linux 64 bit manylinux - os: ubuntu-latest + arch: x86_64 python: 38 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 39 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 310 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - os: ubuntu-latest + arch: x86_64 python: 311 - platform_id: manylinux_x86_64 - manylinux_image: manylinux2014 + builder: manylinux - # MacOS x86_64 - - os: macos-latest + # MacOS x86_64. The macos-13 runner is the last + # Intel-based runner version. At some point we'll + # need to switch to macos-14 and test cross compiling. + - os: macos-13 + arch: x86_64 python: 39 - platform_id: macosx_x86_64 - - os: macos-latest + builder: macosx + - os: macos-13 + arch: x86_64 python: 310 - platform_id: macosx_x86_64 - - os: macos-latest + builder: macosx + - os: macos-13 + arch: x86_64 python: 311 - platform_id: macosx_x86_64 - - # MacOS arm64 - Re-enable after github CI supports M1 runners: - # https://github.com/actions/virtual-environments/issues/2187 - # - # Note: this may also require a revisit of clang / cereal errors. - # - # - os: macos-11 - # python: 39 - # platform_id: macosx_arm64 - # - os: macos-11 - # python: 310 - # platform_id: macosx_arm64 + builder: macosx env: - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014 CIBW_BUILD_VERBOSITY: 3 - CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local" - CIBW_ENVIRONMENT_MACOS: "CC=gcc-12 CXX=g++-12 CFLAGS='-O3 -g -fPIC' CXXFLAGS='-O3 -g -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local" + CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib" CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh - CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.platform_id }} + CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }} CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel} CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs} CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2 @@ -74,28 +67,23 @@ jobs: python -c 'import so3g.smurf.reader; from spt3g import core' && python -m pytest {package}/test steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.10' - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==2.15.0 + python -m pip install cibuildwheel==2.20.0 - name: Build wheel run: | python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: wheels - path: ./wheelhouse/so3g*.whl + name: wheels_cp${{ matrix.python }}-${{ matrix.builder }} + path: ./wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl diff --git a/Dockerfile b/Dockerfile index 540948c..49abf5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # A containerized so3g installation. # Build on spt3g base image -FROM simonsobs/spt3g:0.3-276-gbe879ce +FROM simonsobs/spt3g:0.3-289-g4bd3275 # Set locale ENV LANG C.UTF-8 diff --git a/python/__init__.py b/python/__init__.py index 3b7eab6..790512d 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -1,4 +1,12 @@ import os +import numpy as np + +# Verify that we are using numpy 1.x +npversion = np.version.version +npversplit = npversion.split(".") +if npversplit[0] != "1": + msg = f"so3g requires numpy version 1.x but found version {npversion}" + raise RuntimeError(msg) if os.getenv('DOCS_BUILD') == '1': from ._libso3g_docstring_shells import * diff --git a/requirements.txt b/requirements.txt index 141f73c..b825a6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ astropy matplotlib -numpy +numpy<2 scipy ephem pytz diff --git a/setup.py b/setup.py index 888b019..d614cc7 100644 --- a/setup.py +++ b/setup.py @@ -373,7 +373,7 @@ def readme(): conf["python_requires"] = ">=3.7.0" conf["setup_requires"] = (["wheel", "cmake"],) conf["install_requires"] = [ - "numpy", + "numpy<2", "astropy", "matplotlib", "scipy", diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh index 4ed7edc..5d8bf7f 100755 --- a/wheels/install_deps_linux.sh +++ b/wheels/install_deps_linux.sh @@ -33,7 +33,7 @@ PREFIX=/usr/local pip install --upgrade pip # Install a couple of base packages that are always required -pip install -v cmake wheel +pip install -v cmake wheel setuptools # In order to maximize ABI compatibility with numpy, build with the newest numpy # version containing the oldest ABI version compatible with the python we are using. @@ -59,7 +59,7 @@ CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements # Install Openblas -openblas_version=0.3.21 +openblas_version=0.3.28 openblas_dir=OpenBLAS-${openblas_version} openblas_pkg=${openblas_dir}.tar.gz @@ -83,14 +83,14 @@ tar xzf ${openblas_pkg} \ # Install boost -boost_version=1_80_0 +boost_version=1_86_0 boost_dir=boost_${boost_version} boost_pkg=${boost_dir}.tar.bz2 echo "Fetching boost..." if [ ! -e ${boost_pkg} ]; then - curl -SL "https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/${boost_pkg}" -o "${boost_pkg}" + curl -SL "https://archives.boost.io/release/1.86.0/source/${boost_pkg}" -o "${boost_pkg}" fi echo "Building boost..." diff --git a/wheels/install_deps_osx.sh b/wheels/install_deps_osx.sh index 5104fc5..dab3c07 100755 --- a/wheels/install_deps_osx.sh +++ b/wheels/install_deps_osx.sh @@ -24,8 +24,8 @@ use_gcc=yes # use_gcc=no if [ "x${use_gcc}" = "xyes" ]; then - CC=gcc-12 - CXX=g++-12 + CC=gcc-14 + CXX=g++-14 CFLAGS="-O3 -fPIC" CXXFLAGS="-O3 -fPIC -std=c++14" else @@ -46,14 +46,14 @@ brew install flac # Optionally install gcc if [ "x${use_gcc}" = "xyes" ]; then - brew install gcc@12 + brew install gcc@14 fi # Update pip pip install --upgrade pip # Install a couple of base packages that are always required -pip install -v cmake wheel +pip install -v cmake wheel setuptools # In order to maximize ABI compatibility with numpy, build with the newest numpy # version containing the oldest ABI version compatible with the python we are using. @@ -75,49 +75,37 @@ if [ ${pyver} == "3.11" ]; then fi # Install build requirements. -CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements.txt" "numpy<${numpy_ver}" +CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements.txt" "numpy<${numpy_ver}" "scipy_openblas32" -# Install openblas from the multilib package- the same one numpy uses. +# We use the scipy openblas wheel to get the openblas to use. -if [ "${arch}" = "macosx_arm64" ]; then - openblas_pkg="openblas-v0.3.21-macosx_11_0_arm64-gf_5272328.tar.gz" -else - openblas_pkg="openblas-v0.3.21-macosx_10_9_x86_64-gf_1becaaa.tar.gz" -fi -openblas_url="https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.21/download/${openblas_pkg}" - -if [ ! -e ${openblas_pkg} ]; then - echo "Fetching OpenBLAS..." - curl -SL ${openblas_url} -o ${openblas_pkg} +# First ensure that pkg-config is set to search somewhere +if [ -z "${PKG_CONFIG_PATH}" ]; then + export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" fi -echo "Extracting OpenBLAS" -tar -x -z -v -C "${PREFIX}" --strip-components 2 -f ${openblas_pkg} - -# Install the gfortran (and libgcc) that was used for openblas compilation - -curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg -GFORTRAN_SHA256=$(shasum -a 256 gfortran.dmg) -KNOWN_SHA256="d2d5ca5ba8332d63bbe23a07201c4a0a5d7e09ee56f0298a96775f928c3c4b30 gfortran.dmg" -if [ "$GFORTRAN_SHA256" != "$KNOWN_SHA256" ]; then - echo sha256 mismatch - exit 1 -fi +python3 -c "import scipy_openblas32; print(scipy_openblas32.get_pkg_config())" > ${PKG_CONFIG_PATH}/scipy-openblas.pc -hdiutil attach -mountpoint /Volumes/gfortran gfortran.dmg -sudo installer -pkg /Volumes/gfortran/gfortran.pkg -target / -otool -L /usr/local/gfortran/lib/libgfortran.3.dylib +# To help delocate find the libraries, we copy them into /usr/local +python3 <