From db068474a693a90f209a81d97337864b5a6e1513 Mon Sep 17 00:00:00 2001 From: Karen Hambardzumyan Date: Wed, 20 Apr 2022 20:18:38 +0400 Subject: [PATCH] [feat] M1 Packaging (#16) * All the Mac wheels are being built on custom m1 runner, * We use `arch -x86_64` and `arch -arm64` to switch / emulate architectures, * We first build non-python dependencies and then branch building wheels for each python version, * M1 builds start from python 3.8+ --- .github/workflows/mac/build-bzip2.sh | 11 ++ .github/workflows/mac/build-lz4.sh | 11 ++ .github/workflows/mac/build-rocksdb.sh | 15 +++ .github/workflows/mac/build-snappy.sh | 14 +++ .github/workflows/mac/build-zlib.sh | 12 +++ .github/workflows/mac/build-zstd.sh | 11 ++ .github/workflows/python-package.yml | 133 ++++++++----------------- setup.py | 2 +- 8 files changed, 117 insertions(+), 92 deletions(-) create mode 100755 .github/workflows/mac/build-bzip2.sh create mode 100755 .github/workflows/mac/build-lz4.sh create mode 100755 .github/workflows/mac/build-rocksdb.sh create mode 100755 .github/workflows/mac/build-snappy.sh create mode 100755 .github/workflows/mac/build-zlib.sh create mode 100755 .github/workflows/mac/build-zstd.sh diff --git a/.github/workflows/mac/build-bzip2.sh b/.github/workflows/mac/build-bzip2.sh new file mode 100755 index 0000000..b70df98 --- /dev/null +++ b/.github/workflows/mac/build-bzip2.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +curl https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz -o bzip2-1.0.8.tar.gz +tar zxvf bzip2-1.0.8.tar.gz +cd bzip2-1.0.8/ +make CFLAGS="-fPIC" CXXFLAGS="-fPIC" +make install PREFIX=.. +cd ../ +rm -rf bzip2-1.0.8 bzip2-1.0.8.tar.gz diff --git a/.github/workflows/mac/build-lz4.sh b/.github/workflows/mac/build-lz4.sh new file mode 100755 index 0000000..eaa39c8 --- /dev/null +++ b/.github/workflows/mac/build-lz4.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +curl -L https://github.com/lz4/lz4/archive/v1.9.3.tar.gz -o lz4-1.9.3.tar.gz +tar zxvf lz4-1.9.3.tar.gz +cd lz4-1.9.3 +make CFLAGS="-fPIC" CXXFLAGS="-fPIC" +make PREFIX=$PWD/.. install +cd .. +rm -rf lz4-1.9.3 lz4-1.9.3.tar.gz diff --git a/.github/workflows/mac/build-rocksdb.sh b/.github/workflows/mac/build-rocksdb.sh new file mode 100755 index 0000000..9340a5e --- /dev/null +++ b/.github/workflows/mac/build-rocksdb.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +rm lib/*.dylib +curl -L https://github.com/facebook/rocksdb/archive/6.29.fb.tar.gz -o rocksdb-6.29.fb.tar.gz +tar zxvf rocksdb-6.29.fb.tar.gz +cd rocksdb-6.29.fb +LIBRARY_PATH=$AIM_DEP_DIR/lib PORTABLE=1 make shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j12 +strip -S librocksdb.dylib +install_name_tool -id @rpath/librocksdb.dylib librocksdb.dylib +cp librocksdb.dylib $AIM_DEP_DIR/lib +LIBRARY_PATH=$AIM_DEP_DIR/lib PREFIX="$AIM_DEP_DIR" PORTABLE=1 make install-headers PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j12 +cd .. +rm -rf rocksdb-6.29.fb rocksdb-6.29.fb.tar.gz diff --git a/.github/workflows/mac/build-snappy.sh b/.github/workflows/mac/build-snappy.sh new file mode 100755 index 0000000..6e973ee --- /dev/null +++ b/.github/workflows/mac/build-snappy.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz -o snappy-1.1.8.tar.gz +tar zxvf snappy-1.1.8.tar.gz +cd snappy-1.1.8 +mkdir build +cd build +$CMAKE -DCMAKE_INSTALL_PREFIX=../.. CFLAGS="-fPIC" CXXFLAGS="-fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. +make +make install +cd ../.. +rm -rf snappy-1.1.8 snappy-1.1.8.tar.gz diff --git a/.github/workflows/mac/build-zlib.sh b/.github/workflows/mac/build-zlib.sh new file mode 100755 index 0000000..3b67b7a --- /dev/null +++ b/.github/workflows/mac/build-zlib.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +curl -L https://www.zlib.net/fossils/zlib-1.2.11.tar.gz -o zlib-1.2.11.tar.gz +tar zxvf zlib-1.2.11.tar.gz +cd zlib-1.2.11/ +./configure +make CFLAGS="-fPIC" CXXFLAGS="-fPIC" +make install prefix=.. +cd ../ +rm -rf zlib-1.2.11 zlib-1.2.11.tar.gz diff --git a/.github/workflows/mac/build-zstd.sh b/.github/workflows/mac/build-zstd.sh new file mode 100755 index 0000000..916d904 --- /dev/null +++ b/.github/workflows/mac/build-zstd.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cd $AIM_DEP_DIR +curl -L https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -o zstd-1.1.3.tar.gz +tar zxvf zstd-1.1.3.tar.gz +cd zstd-1.1.3 +make CFLAGS="-fPIC" CXXFLAGS="-fPIC" +make install PREFIX=$PWD/.. +cd .. +rm -rf zstd-1.1.3 zstd-1.1.3.tar.gz diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 606f378..f2262ef 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -72,128 +72,79 @@ jobs: run: | python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" manylinux_dist/* - macos-dist: - runs-on: ${{matrix.os}} + macos-deps: + runs-on: m1 strategy: - fail-fast: false + fail-fast: true matrix: - python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' ] - os: [ 'macos-latest' ] - name: Python ${{ matrix.python-version }} on ${{ matrix.os }} build + arch: ['arm64', 'x86_64'] + name: Preparing dependencies for ${{ matrix.arch }} build env: MACOSX_DEPLOYMENT_TARGET: 10.14 - AIM_DEP_DIR: /tmp/aimrocks_deps + AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} + CMAKE: /opt/homebrew/bin/cmake steps: - name: Preparing Build Dir for Dependencies run: | mkdir -p $AIM_DEP_DIR + rm -rf $AIM_DEP_DIR/* + + - name: Checkout sources + uses: actions/checkout@v2 - name: Building ZLib run: | - cd $AIM_DEP_DIR - curl -L https://www.zlib.net/fossils/zlib-1.2.11.tar.gz -o zlib-1.2.11.tar.gz - tar zxvf zlib-1.2.11.tar.gz - cd zlib-1.2.11/ - ./configure - make CFLAGS="-fPIC" CXXFLAGS="-fPIC" - make install prefix=.. - cd ../ - rm -rf zlib-1.2.11 zlib-1.2.11.tar.gz + pwd + ls -lhatr .github/workflows/mac + arch -${{matrix.arch}} ./.github/workflows/mac/build-zlib.sh - name: Building BZip2 run: | - cd $AIM_DEP_DIR - curl https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz -o bzip2-1.0.8.tar.gz - tar zxvf bzip2-1.0.8.tar.gz - cd bzip2-1.0.8/ - make CFLAGS="-fPIC" CXXFLAGS="-fPIC" - make install PREFIX=.. - cd ../ - rm -rf bzip2-1.0.8 bzip2-1.0.8.tar.gz + arch -${{matrix.arch}} ./.github/workflows/mac/build-bzip2.sh - name: Building LZ4 run: | - cd $AIM_DEP_DIR - curl -L https://github.com/lz4/lz4/archive/v1.9.3.tar.gz -o lz4-1.9.3.tar.gz - tar zxvf lz4-1.9.3.tar.gz - cd lz4-1.9.3 - make CFLAGS="-fPIC" CXXFLAGS="-fPIC" - make PREFIX=$PWD/.. install - cd .. - rm -rf lz4-1.9.3 lz4-1.9.3.tar.gz + arch -${{matrix.arch}} ./.github/workflows/mac/build-lz4.sh - name: Building Snappy run: | - cd $AIM_DEP_DIR - curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz -o snappy-1.1.8.tar.gz - tar zxvf snappy-1.1.8.tar.gz - cd snappy-1.1.8 - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=../.. CFLAGS="-fPIC" CXXFLAGS="-fPIC" -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. - make - make install - cd ../.. - rm -rf snappy-1.1.8 snappy-1.1.8.tar.gz + arch -${{matrix.arch}} ./.github/workflows/mac/build-snappy.sh - name: Building ZSTD run: | - cd $AIM_DEP_DIR - curl -L https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -o zstd-1.1.3.tar.gz - tar zxvf zstd-1.1.3.tar.gz - cd zstd-1.1.3 - make CFLAGS="-fPIC" CXXFLAGS="-fPIC" - make install PREFIX=$PWD/.. - cd .. - rm -rf zstd-1.1.3 zstd-1.1.3.tar.gz - - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 + arch -${{matrix.arch}} ./.github/workflows/mac/build-zstd.sh - name: Building RocksDB run: | - # rocksdb - cd $AIM_DEP_DIR - rm lib/*.dylib - curl -L https://github.com/facebook/rocksdb/archive/6.29.fb.tar.gz -o rocksdb-6.29.fb.tar.gz - tar zxvf rocksdb-6.29.fb.tar.gz - cd rocksdb-6.29.fb - LIBRARY_PATH=$AIM_DEP_DIR/lib PORTABLE=1 make shared_lib PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j6 - strip -S librocksdb.dylib - install_name_tool -id @rpath/librocksdb.dylib librocksdb.dylib - cp librocksdb.dylib $AIM_DEP_DIR/lib - LIBRARY_PATH=$AIM_DEP_DIR/lib PREFIX="$AIM_DEP_DIR" PORTABLE=1 make install-headers PLATFORM_SHARED_VERSIONED=false EXTRA_CXXFLAGS="-fPIC -I$AIM_DEP_DIR/include" USE_RTTI=0 DEBUG_LEVEL=0 -j6 - cd .. - rm -rf rocksdb-6.29.fb rocksdb-6.29.fb.tar.gz - - - name: Install dev dependencies - run: | - python -m pip install -r requirements.dev.txt - - - name: Build bdist wheel - run: | - python setup.py bdist_wheel -d dist --plat-name=macosx_10_14_x86_64 - - - name: Installing from bdist - run: | - python -m pip install dist/*.whl + arch -${{matrix.arch}} ./.github/workflows/mac/build-rocksdb.sh - - name: Inspecting bdist files - run: | - python -m pip show -f aimrocks - - name: Test bdist + macos-dist: + runs-on: m1 + needs: macos-deps + strategy: + fail-fast: true + matrix: + python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' ] + arch: ['arm64', 'x86_64'] + exclude: + - arch: 'arm64' + python-version: 3.6 + - arch: 'arm64' + python-version: 3.7 + # requirement for custom runners: having cmake and python installed + name: Python ${{ matrix.python-version }} for ${{ matrix.arch }} build + env: + MACOSX_DEPLOYMENT_TARGET: 10.14 + AIM_DEP_DIR: /tmp/run/${{ github.run_number }}/${{ matrix.arch }} + PYTHON: /opt/conda/${{ matrix.arch }}/envs/py${{ matrix.python-version }}/bin/python + steps: + - name: Build and test wheels run: | - python -m pytest tests || true + arch -${{matrix.arch}} $PYTHON -m build - name: Publish wheel env: PYPI_PASSWORD: ${{ secrets.pypi_password }} run: | - python -m twine upload -u __token__ -p "${PYPI_PASSWORD}" dist/* + $PYTHON -m twine upload --verbose -u __token__ -p "${PYPI_PASSWORD}" dist/*.whl diff --git a/setup.py b/setup.py index fdf04ff..a6ba6e3 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ setup( name="aimrocks", - version='0.2.0', + version='0.2.1', description='RocksDB wrapper implemented in Cython.', setup_requires=['setuptools>=25', 'Cython>=3.0.0a9'], packages=find_packages('./src'),