-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add CI for pybind wrapper use macos-14 runner update tag disable macos-14 remove integration tests skip pp37 adapt sdist fix style fix style revert cibuildwheel changes try to fix sdist move keyvi source out of src install pytest add more match bindings move module import switch to scikit_build remove license file
- Loading branch information
1 parent
1223704
commit 1780596
Showing
14 changed files
with
784 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| name: Python Pybind cibuildwheel | ||
|
|
||
| on: | ||
| push: | ||
| branches: [master, release-*] | ||
| pull_request: | ||
| branches: [master] | ||
| workflow_dispatch: | ||
|
|
||
| jobs: | ||
| build_wheels: | ||
| name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }} | ||
| runs-on: ${{ matrix.os }} | ||
| strategy: | ||
| matrix: | ||
| os: [ubuntu-22.04] | ||
| # separate archs, so they use individual caches | ||
| arch: ["x86_64", "arm64"] | ||
| flavor: ["cpython", "pypy"] | ||
| # separate musl and many on linux, for mac we just skip one of those | ||
| target: ["many", "musl"] | ||
| exclude: | ||
| - os: ubuntu-22.04 | ||
| target: musl | ||
| flavor: pypy | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up QEMU | ||
| if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }} | ||
| uses: docker/setup-qemu-action@v3 | ||
| with: | ||
| platforms: all | ||
| - name: ccache | ||
| uses: hendrikmuhs/[email protected] | ||
| with: | ||
| key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python | ||
|
|
||
| - name: Sets env for x86_64 | ||
| run: | | ||
| echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV | ||
| echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV | ||
| if: matrix.arch == 'x86_64' | ||
|
|
||
| - name: Sets env for arm64 | ||
| run: | | ||
| echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV | ||
| echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV | ||
| if: matrix.arch == 'arm64' | ||
|
|
||
| - name: Skip manylinux for musllinux target | ||
| if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }} | ||
| run: | | ||
| echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV | ||
| - name: Skip musllinux for manylinux target | ||
| if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }} | ||
| run: | | ||
| echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV | ||
| - name: Skip pypy for cpython | ||
| if: ${{ matrix.flavor == 'cpython' }} | ||
| run: | | ||
| echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV | ||
| - name: Skip cpython for pypy | ||
| if: ${{ matrix.flavor == 'pypy' }} | ||
| run: | | ||
| echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV | ||
| - name: install mac dependencies | ||
| if: ${{ runner.os == 'macOS' }} | ||
| # 2nd command: workaround https://github.com/actions/setup-python/issues/577 | ||
| run: | | ||
| brew update && \ | ||
| brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \ | ||
| brew install ccache | ||
| - name: install mac dependencies X86_64 | ||
| if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }} | ||
| run: | | ||
| brew update && \ | ||
| brew install zlib snappy boost | ||
| - name: install mac dependencies arm64 | ||
| if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }} | ||
| run: | | ||
| set -e | ||
| echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV | ||
| echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV | ||
| echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV | ||
| mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew | ||
| PACKAGES=( icu4c xz lz4 zstd zlib snappy boost ) | ||
| for PACKAGE in "${PACKAGES[@]}" | ||
| do | ||
| response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_sonoma $PACKAGE | grep Downloaded ) | ||
| download_path=$(echo $response | xargs -n 1 | tail -1) | ||
| arm64-homebrew/bin/brew reinstall -vd $download_path | ||
| done | ||
| arm64-homebrew/bin/brew config | ||
| ls /Users/runner/work/keyvi/keyvi/arm64-homebrew | ||
| - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }} | ||
| uses: pypa/[email protected] | ||
| env: | ||
| # Skip CPython 3.6 and CPython 3.7 | ||
| CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* pp37-* | ||
|
|
||
| # skip testing all python versions on linux arm, only test 3.12 | ||
| # skip tests on pypy, currently fails for indexer tests | ||
| CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*" | ||
|
|
||
| # (many)linux custom docker images | ||
| CIBW_MANYLINUX_X86_64_IMAGE: "keyvidev/manylinux-builder-x86_64" | ||
| CIBW_MANYLINUX_AARCH64_IMAGE: "keyvidev/manylinux-builder-aarch64" | ||
| CIBW_MUSLLINUX_X86_64_IMAGE: "keyvidev/musllinux-builder-x86_64" | ||
| CIBW_MUSLLINUX_AARCH64_IMAGE: "keyvidev/musllinux-builder-aarch64" | ||
|
|
||
| # ccache using path | ||
| CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH | ||
| CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf | ||
|
|
||
| # python dependencies | ||
| CIBW_BEFORE_BUILD: pip install -r python/requirements.txt | ||
|
|
||
| # testing | ||
| CIBW_TEST_REQUIRES: pytest | ||
| CIBW_TEST_COMMAND: > | ||
| python -m pytest {package}/tests | ||
| # for debugging set this to 1,2 or 3 | ||
| # CIBW_BUILD_VERBOSITY: 2 | ||
| with: | ||
| package-dir: python-pybind | ||
|
|
||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }} | ||
| path: ./wheelhouse/*.whl | ||
|
|
||
| build_sdist: | ||
| name: sdist | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: install Linux deps | ||
| run: | | ||
| sudo apt-get update && \ | ||
| sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache | ||
| - name: ccache | ||
| uses: hendrikmuhs/[email protected] | ||
| with: | ||
| key: ${{ matrix.os }}-sdist-python | ||
|
|
||
| - name: Build SDist | ||
| run: | | ||
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | ||
| cd python-pybind && \ | ||
| python -m pip install . && \ | ||
| python setup.py sdist -d wheelhouse && \ | ||
| python -m pip install wheelhouse/*.tar.gz -v && \ | ||
| python -m pip install pytest && \ | ||
| python -m pytest tests && \ | ||
| python -m pip uninstall -y keyvi_pybind11 | ||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: artifact-sdist | ||
| path: python-pybind/wheelhouse/*.tar.gz | ||
|
|
||
| upload_all: | ||
| needs: [build_wheels, build_sdist] | ||
| runs-on: ubuntu-latest | ||
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | ||
| steps: | ||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| pattern: artifact-* | ||
| merge-multiple: true | ||
| path: dist | ||
|
|
||
| - uses: pypa/gh-action-pypi-publish@release/v1 | ||
| with: | ||
| password: ${{ secrets.pypi_password }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.