From 3cdad9b5d1684c0d08dc8e954e6ef0b7593cde3d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sun, 24 Mar 2024 07:54:32 +0800 Subject: [PATCH] Use manylinux in CI test (#692) --- .../workflows/aarch64-linux-gnu-shared.yaml | 2 +- .../workflows/aarch64-linux-gnu-static.yaml | 2 +- .github/workflows/android.yaml | 31 +++-- .github/workflows/arm-linux-gnueabihf.yaml | 2 +- .github/workflows/build-xcframework.yaml | 33 +++-- .github/workflows/linux.yaml | 129 +++++++++++------- .github/workflows/riscv64-linux.yaml | 2 +- .github/workflows/windows-x64.yaml | 2 +- .github/workflows/windows-x86.yaml | 2 +- CMakeLists.txt | 2 +- 10 files changed, 121 insertions(+), 86 deletions(-) diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 34ad56fd2..01d601aba 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -179,7 +179,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 69916076b..27579ea1e 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -170,7 +170,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index f31297513..dd92a6d91 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -101,28 +101,33 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface - cd huggingface - git lfs pull + cd huggingface + git lfs pull - cp -v ../sherpa-onnx-*-android.tar.bz2 ./ + cp -v ../sherpa-onnx-*-android.tar.bz2 ./ - git status - git lfs track "*.bz2" + git status + git lfs track "*.bz2" - git add . + git add . - git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-android.tar.bz2" + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-android.tar.bz2" - git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release android libs if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index 7bc58d1cf..76ad9fcf8 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -193,7 +193,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml index 4027030e2..be4e3666b 100644 --- a/.github/workflows/build-xcframework.yaml +++ b/.github/workflows/build-xcframework.yaml @@ -78,31 +78,36 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface - cd huggingface - git lfs pull + cd huggingface + git lfs pull - cp -v ../sherpa-onnx-*-ios.tar.bz2 ./ + cp -v ../sherpa-onnx-*-ios.tar.bz2 ./ - git status - git lfs track "*.bz2" + git status + git lfs track "*.bz2" - git add . + git add . - git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-ios.tar.bz2" + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-ios.tar.bz2" - git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release xcframework - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 037c7977b..a60b3e430 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -45,77 +45,66 @@ concurrency: jobs: linux: - name: ${{ matrix.os }} ${{ matrix.build_type }} ${{ matrix.shared_lib }} ${{ matrix.gcc_version }} + name: ${{ matrix.build_type }} ${{ matrix.shared_lib }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-22.04] + os: [ubuntu-latest] build_type: [Release, Debug] shared_lib: [ON, OFF] - # see https://github.com/egor-tensin/setup-gcc - # 7-11 for ubuntu 20.04 - # 9-12 for ubuntu 22.04 - gcc_version: ["7", "8", "9", "10", "11", "12", "13"] - exclude: - - os: ubuntu-20.04 - gcc_version: "12" - - os: ubuntu-22.04 - gcc_version: "7" - - os: ubuntu-22.04 - gcc_version: "8" steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up GCC ${{ matrix.gcc_version }} - uses: egor-tensin/setup-gcc@v1 - with: - version: ${{ matrix.gcc_version }} - platform: x64 - - - name: Display gcc version ${{ matrix.gcc_version }} + - name: Display PWD shell: bash run: | - which gcc - gcc --version - - which g++ - g++ --version + echo "pwd: $PWD" + ls -lh - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 + - name: Build sherpa-onnx + uses: addnab/docker-run-action@v3 with: - key: ${{ matrix.os }}-${{ matrix.build_type }}-shared-${{ matrix.shared_lib }}-gcc-${{ matrix.gcc_version }} + image: quay.io/pypa/manylinux2014_x86_64 + options: | + --volume ${{ github.workspace }}/:/home/runner/work/sherpa-onnx/sherpa-onnx + shell: bash + run: | + uname -a + gcc --version + cmake --version + cat /etc/*release + id + pwd - - name: Configure CMake - shell: bash - run: | - export CMAKE_CXX_COMPILER_LAUNCHER=ccache - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cmake --version + cd /home/runner/work/sherpa-onnx/sherpa-onnx - mkdir build - cd build - cmake -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} -DCMAKE_INSTALL_PREFIX=./install .. + git clone --depth 1 https://github.com/alsa-project/alsa-lib + pushd alsa-lib + ./gitcompile + popd - - name: Build sherpa-onnx for ubuntu - shell: bash - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH + export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs + + mkdir build + cd build - cd build - make -j2 - make install + cmake -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} -DCMAKE_INSTALL_PREFIX=./install .. - ls -lh lib - ls -lh bin + make -j2 + make install + + ls -lh lib + ls -lh bin - name: Display dependencies of sherpa-onnx for linux shell: bash run: | + sudo chown -R $USER ./build ls -lh build/bin ls -lh build/_deps/onnxruntime-src/lib/ @@ -130,9 +119,8 @@ jobs: readelf -d build/bin/sherpa-onnx - uses: actions/upload-artifact@v4 - if: matrix.shared_lib == 'OFF' && matrix.build_type == 'Release' with: - name: release-static-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.shared_lib }}-${{ matrix.gcc_version }} + name: release-${{ matrix.build_type }}-${{ matrix.shared_lib }} path: build/bin/* - name: Test transducer kws @@ -211,10 +199,17 @@ jobs: - name: Copy files shell: bash + if: matrix.build_type == 'Release' run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-x64 + if [[ ${{ matrix.shared_lib }} == 'ON' ]]; then + suffix=shared + else + suffix=static + fi + + dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-x64-$suffix mkdir $dst cp -a build/install/bin $dst/ @@ -225,16 +220,46 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst + - name: Publish to huggingface + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && matrix.build_type == 'Release' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface + + cd huggingface + git lfs pull + mkdir -p linux-x64 + + cp -v ../sherpa-onnx-*.tar.bz2 ./linux-x64 + + git status + git lfs track "*.bz2" + + git add . + + git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}" + + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main + - name: Release pre-compiled binaries and libs for linux x64 - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-20.04' && matrix.gcc_version == '7' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.build_type == 'Release' uses: svenstaro/upload-release-action@v2 with: file_glob: true overwrite: true - file: sherpa-onnx-*linux-x64.tar.bz2 + file: sherpa-onnx-*.tar.bz2 - uses: actions/upload-artifact@v4 with: - name: tts-generated-test-files-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.shared_lib }}-${{ matrix.gcc_version }} + name: tts-generated-test-files-${{ matrix.build_type }}-${{ matrix.shared_lib }} path: tts diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml index b887f8949..b1008b514 100644 --- a/.github/workflows/riscv64-linux.yaml +++ b/.github/workflows/riscv64-linux.yaml @@ -225,7 +225,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 5ea29c816..46daea36b 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -154,7 +154,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index b2bd25f3b..1ed8ea0a0 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -157,7 +157,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 diff --git a/CMakeLists.txt b/CMakeLists.txt index a4dbd72ed..45e359e4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) project(sherpa-onnx) -set(SHERPA_ONNX_VERSION "1.9.12") +set(SHERPA_ONNX_VERSION "1.9.13") # Disable warning about #