From 6962d749ae3ceb126e5146d4ab66e35ccc093c34 Mon Sep 17 00:00:00 2001 From: John Blischak Date: Tue, 10 Oct 2023 10:23:10 -0400 Subject: [PATCH] Nightly build for macOS (#579) * Nightly build for macOS * Switch to pip for installation * Bump macOS image from 11 to 12 https://github.com/TileDB-Inc/TileDB/pull/4367 * Pin fsspec<2023.3.0 https://github.com/TileDB-Inc/TileDB-VCF/commit/a53ef3308c5a7380e9102260e4327562aeaf51be * Install Pythons dependencies with PyPI wheels on macOS * Use $CONDA instead of $CONDA_PREFIX --- .github/workflows/nightly.yml | 61 ++++++++++++++++++++++++++++---- ci/azure-linux_mac.yml | 4 +-- ci/nightly/build-tiledbvcf-py.sh | 15 ++++++-- ci/nightly/requirements-test.txt | 4 +++ ci/nightly/requirements.txt | 8 +++++ 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 ci/nightly/requirements-test.txt create mode 100644 ci/nightly/requirements.txt diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ec4115ddf..52dca8c3d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,9 @@ name: Nightly build against nightly libtiledb # * Installs libtiledb and libtiledbvcf into ./install/ # * Manually updates library path so that tiledbvcf-py can link to # the shared objects in ./install/ -# * conda is only used to install Python dependencies +# * The Python dependencies are handled differently per OS +# * linux: conda binaries +# * osx: PyPI wheels on: push: paths: @@ -17,14 +19,13 @@ on: - cron: "0 3 * * *" # Every night at 3 AM UTC (10 PM EST; 11 PM EDT) workflow_dispatch: jobs: - build: - runs-on: ${{ matrix.os }} - name: ${{ matrix.os }}-libtiledb-${{ matrix.libtiledb_version }} + linux: + runs-on: ubuntu-latest + name: linux-libtiledb-${{ matrix.libtiledb_version }} strategy: fail-fast: false matrix: libtiledb_version: [release-2.17, dev] - os: [ubuntu-latest] steps: - name: Clone TileDB uses: actions/checkout@v3 @@ -44,7 +45,7 @@ jobs: - name: Confirm linking run: | ldd install/lib/libtiledb.so - ldd install/lib/libtiledbvcf.so | grep tile + ldd install/lib/libtiledbvcf.so - name: Install bcftools (for tests) run: sudo apt install bcftools - name: Test libtiledbvcf @@ -58,11 +59,57 @@ jobs: # need -l to activate conda env shell: bash -l {0} run: bash TileDB-VCF/ci/nightly/build-tiledbvcf-py.sh + osx: + runs-on: macos-12 + name: osx-libtiledb-${{ matrix.libtiledb_version }} + strategy: + fail-fast: false + matrix: + libtiledb_version: [release-2.17, dev] + env: + MACOSX_DEPLOYMENT_TARGET: 10.15 + steps: + - name: Clone TileDB + uses: actions/checkout@v3 + with: + repository: TileDB-Inc/TileDB + ref: ${{ matrix.libtiledb_version }} + path: TileDB + - name: Clone TileDB-VCF + uses: actions/checkout@v3 + with: + path: TileDB-VCF + fetch-depth: 0 # fetch everything for python setuptools_scm + - name: Build libtiledb + run: bash TileDB-VCF/ci/nightly/build-libtiledb.sh + - name: Setup to build htslib from source + run: brew install autoconf automake + - name: Build libtiledbvcf + run: bash TileDB-VCF/ci/nightly/build-libtiledbvcf.sh + - name: Confirm linking + run: | + otool -L install/lib/libtiledb.dylib + otool -L install/lib/libtiledbvcf.dylib + - name: Install bcftools (for tests) + run: brew install bcftools + - name: Test libtiledbvcf + run: bash TileDB-VCF/ci/nightly/test-libtiledbvcf.sh + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --prefer-binary \ + -r TileDB-VCF/ci/nightly/requirements.txt \ + -r TileDB-VCF/ci/nightly/requirements-test.txt + - name: Build (and test) tiledbvcf-py + run: bash TileDB-VCF/ci/nightly/build-tiledbvcf-py.sh issue: permissions: issues: write runs-on: ubuntu-latest - needs: build + needs: [linux, osx] if: ( failure() || cancelled() ) && github.repository_owner == 'TileDB-Inc' && github.event_name == 'schedule' steps: - uses: actions/checkout@v3 diff --git a/ci/azure-linux_mac.yml b/ci/azure-linux_mac.yml index 852461a3d..fa3821e02 100755 --- a/ci/azure-linux_mac.yml +++ b/ci/azure-linux_mac.yml @@ -153,7 +153,7 @@ steps: fi eval "$(micromamba shell hook --shell bash)" micromamba --version - + # Create and activate the conda environment pushd apis/python micromamba create -f conda-env.yml -y @@ -213,4 +213,4 @@ steps: cat $f done; condition: failed() # only run this job if the build step failed - displayName: "Print log files (failed build only)" \ No newline at end of file + displayName: "Print log files (failed build only)" diff --git a/ci/nightly/build-tiledbvcf-py.sh b/ci/nightly/build-tiledbvcf-py.sh index d5a16914c..a5ace6359 100644 --- a/ci/nightly/build-tiledbvcf-py.sh +++ b/ci/nightly/build-tiledbvcf-py.sh @@ -5,13 +5,22 @@ set -ex # ./TileDB-VCF/apis/python and libtiledbvcf shared library installed in # $GITHUB_WORKSPACE/install/ -export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install/lib:${LD_LIBRARY_PATH-} -echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" +OS=$(uname) +echo "OS: $OS" +if [[ $OS == Linux ]] +then + export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install/lib:${LD_LIBRARY_PATH-} + echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" +elif [[ $OS == Darwin ]] +then + export DYLD_LIBRARY_PATH=$GITHUB_WORKSPACE/install/lib:${DYLD_LIBRARY_PATH-} + echo "DYLD_LIBRARY_PATH: $DYLD_LIBRARY_PATH" +fi export LIBTILEDBVCF_PATH=$GITHUB_WORKSPACE/install/ cd TileDB-VCF/apis/python -python setup.py install +python -m pip install . python -c "import tiledbvcf; print(tiledbvcf.version)" pytest diff --git a/ci/nightly/requirements-test.txt b/ci/nightly/requirements-test.txt new file mode 100644 index 000000000..60a46fe05 --- /dev/null +++ b/ci/nightly/requirements-test.txt @@ -0,0 +1,4 @@ +dask +distributed +pytest +tiledb diff --git a/ci/nightly/requirements.txt b/ci/nightly/requirements.txt new file mode 100644 index 000000000..e6b453e63 --- /dev/null +++ b/ci/nightly/requirements.txt @@ -0,0 +1,8 @@ +numpy +pandas +pyarrow==11 +pybind11 +setuptools +setuptools_scm +setuptools_scm_git_archive +wheel