python-cibuildwheel-pybind.yml

name: Python Pybind cibuildwheel

on:
  push:
    branches: [master, release-*]
  pull_request:
    branches: [master]
  workflow_dispatch:

jobs:
  build_wheels:
    name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-22.04]
        # separate archs, so they use individual caches
        arch: ["x86_64", "arm64"]
        flavor: ["cpython", "pypy"]
        # separate musl and many on linux, for mac we just skip one of those
        target: ["many", "musl"]
        exclude:
          - os: ubuntu-22.04
            target: musl
            flavor: pypy
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
        uses: docker/setup-qemu-action@v3
        with:
          platforms: all
      - name: ccache
        uses: hendrikmuhs/ccache-action@v1.2.11
        with:
          key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python

      - name: Sets env for x86_64
        run: |
          echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
          echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
        if: matrix.arch == 'x86_64'

      - name: Sets env for arm64
        run: |
          echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
          echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
        if: matrix.arch == 'arm64'

      - name: Skip manylinux for musllinux target
        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
        run: |
          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV

      - name: Skip musllinux for manylinux target
        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
        run: |
          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV

      - name: Skip pypy for cpython
        if: ${{ matrix.flavor == 'cpython' }}
        run: |
          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV

      - name: Skip cpython for pypy
        if: ${{ matrix.flavor == 'pypy' }}
        run: |
          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV

      - name: install mac dependencies
        if: ${{ runner.os == 'macOS' }}
        # 2nd command: workaround https://github.com/actions/setup-python/issues/577
        run: |
          brew update && \
          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
          brew install ccache

      - name: install mac dependencies X86_64
        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
        run: |
          brew update && \
          brew install zlib snappy boost

      - name: install mac dependencies arm64
        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
        run: |
          set -e
          echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV
          echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
          mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
          PACKAGES=( icu4c xz lz4 zstd zlib snappy boost )
          for PACKAGE in "${PACKAGES[@]}"
          do
            response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_sonoma $PACKAGE | grep Downloaded )
            download_path=$(echo $response | xargs -n 1 | tail -1)
            arm64-homebrew/bin/brew reinstall -vd $download_path
          done
          arm64-homebrew/bin/brew config
          ls /Users/runner/work/keyvi/keyvi/arm64-homebrew

      - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
        uses: pypa/cibuildwheel@v2.17.0
        env:
          # Skip CPython 3.6 and CPython 3.7
          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* pp37-*

          # skip testing all python versions on linux arm, only test 3.12
          # skip tests on pypy, currently fails for indexer tests
          CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"

          # (many)linux custom docker images
          CIBW_MANYLINUX_X86_64_IMAGE: "keyvidev/manylinux-builder-x86_64"
          CIBW_MANYLINUX_AARCH64_IMAGE: "keyvidev/manylinux-builder-aarch64"
          CIBW_MUSLLINUX_X86_64_IMAGE: "keyvidev/musllinux-builder-x86_64"
          CIBW_MUSLLINUX_AARCH64_IMAGE: "keyvidev/musllinux-builder-aarch64"

          # ccache using path
          CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH
          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf

          # python dependencies
          CIBW_BEFORE_BUILD: pip install -r python/requirements.txt

          # testing
          CIBW_TEST_REQUIRES: pytest
          CIBW_TEST_COMMAND: >
            python -m pytest {package}/tests

          # for debugging set this to 1,2 or 3
          # CIBW_BUILD_VERBOSITY: 2
        with:
          package-dir: python-pybind

      - uses: actions/upload-artifact@v4
        with:
          name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }}
          path: ./wheelhouse/*.whl

  build_sdist:
    name: sdist
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: install Linux deps
        run: |
          sudo apt-get update && \
          sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
      - name: ccache
        uses: hendrikmuhs/ccache-action@v1.2.11
        with:
          key: ${{ matrix.os }}-sdist-python

      - name: Build SDist
        run: |
          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
          cd python-pybind && \
          python -m pip install . && \
          python setup.py sdist -d wheelhouse && \
          python -m pip install wheelhouse/*.tar.gz -v && \
          python -m pip install pytest && \
          python -m pytest tests && \
          python -m pip uninstall -y keyvi_pybind11

      - uses: actions/upload-artifact@v4
        with:
          name: artifact-sdist
          path: python-pybind/wheelhouse/*.tar.gz

  upload_all:
    needs: [build_wheels, build_sdist]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: artifact-*
          merge-multiple: true
          path: dist

      - uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.pypi_password }}