Use manylinux in CI test (#692)

k2-fsa · Mar 23, 2024 · 3cdad9b · 3cdad9b
1 parent e60c897
commit 3cdad9b
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 86 deletions.
diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml
@@ -179,7 +179,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml
@@ -170,7 +170,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml
@@ -101,28 +101,33 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "Fangjun Kuang"
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Fangjun Kuang"
 
-          GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
+            GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
 
-          cd huggingface
-          git lfs pull
+            cd huggingface
+            git lfs pull
 
-          cp -v ../sherpa-onnx-*-android.tar.bz2 ./
+            cp -v ../sherpa-onnx-*-android.tar.bz2 ./
 
-          git status
-          git lfs track "*.bz2"
+            git status
+            git lfs track "*.bz2"
 
-          git add .
+            git add .
 
-          git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-android.tar.bz2"
+            git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-android.tar.bz2"
 
-          git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-libs main
+            git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-libs main
 
       - name: Release android libs
         if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/')

diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml
@@ -193,7 +193,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml
@@ -78,31 +78,36 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "Fangjun Kuang"
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Fangjun Kuang"
 
-          GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
+            GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
 
-          cd huggingface
-          git lfs pull
+            cd huggingface
+            git lfs pull
 
-          cp -v ../sherpa-onnx-*-ios.tar.bz2 ./
+            cp -v ../sherpa-onnx-*-ios.tar.bz2 ./
 
-          git status
-          git lfs track "*.bz2"
+            git status
+            git lfs track "*.bz2"
 
-          git add .
+            git add .
 
-          git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-ios.tar.bz2"
+            git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}-ios.tar.bz2"
 
-          git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-libs main
+            git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-libs main
 
       - name: Release xcframework
-        if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
         uses: svenstaro/upload-release-action@v2
         with:
           file_glob: true

diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml
@@ -45,77 +45,66 @@ concurrency:
 
 jobs:
   linux:
-    name: ${{ matrix.os }} ${{ matrix.build_type }} ${{ matrix.shared_lib }} ${{ matrix.gcc_version }}
+    name: ${{ matrix.build_type }} ${{ matrix.shared_lib }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-latest]
         build_type: [Release, Debug]
         shared_lib: [ON, OFF]
-        # see https://github.com/egor-tensin/setup-gcc
-        # 7-11 for ubuntu 20.04
-        # 9-12 for ubuntu 22.04
-        gcc_version: ["7", "8", "9", "10", "11", "12", "13"]
-        exclude:
-          - os: ubuntu-20.04
-            gcc_version: "12"
-          - os: ubuntu-22.04
-            gcc_version: "7"
-          - os: ubuntu-22.04
-            gcc_version: "8"
 
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Set up GCC ${{ matrix.gcc_version }}
-        uses: egor-tensin/setup-gcc@v1
-        with:
-          version: ${{ matrix.gcc_version }}
-          platform: x64
-
-      - name: Display gcc version ${{ matrix.gcc_version }}
+      - name: Display PWD
         shell: bash
         run: |
-          which gcc
-          gcc --version
-
-          which g++
-          g++ --version
+          echo "pwd: $PWD"
+          ls -lh
 
-      - name: ccache
-        uses: hendrikmuhs/ccache-action@v1.2
+      - name: Build sherpa-onnx
+        uses: addnab/docker-run-action@v3
         with:
-          key: ${{ matrix.os }}-${{ matrix.build_type }}-shared-${{ matrix.shared_lib }}-gcc-${{ matrix.gcc_version }}
+            image: quay.io/pypa/manylinux2014_x86_64
+            options: |
+              --volume ${{ github.workspace }}/:/home/runner/work/sherpa-onnx/sherpa-onnx
+            shell: bash
+            run: |
+              uname -a
+              gcc --version
+              cmake --version
+              cat /etc/*release
+              id
+              pwd
 
-      - name: Configure CMake
-        shell: bash
-        run: |
-          export CMAKE_CXX_COMPILER_LAUNCHER=ccache
-          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
-          cmake --version
+              cd /home/runner/work/sherpa-onnx/sherpa-onnx
 
-          mkdir build
-          cd build
-          cmake -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} -DCMAKE_INSTALL_PREFIX=./install ..
+              git clone --depth 1 https://github.com/alsa-project/alsa-lib
+              pushd alsa-lib
+              ./gitcompile
+              popd
 
-      - name: Build sherpa-onnx for ubuntu
-        shell: bash
-        run: |
-          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+              export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH
+              export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs
+
+              mkdir build
+              cd build
 
-          cd build
-          make -j2
-          make install
+              cmake -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} -DCMAKE_INSTALL_PREFIX=./install ..
 
-          ls -lh lib
-          ls -lh bin
+              make -j2
+              make install
+
+              ls -lh lib
+              ls -lh bin
 
       - name: Display dependencies of sherpa-onnx for linux
         shell: bash
         run: |
+          sudo chown -R $USER ./build
           ls -lh build/bin
           ls -lh build/_deps/onnxruntime-src/lib/
 
@@ -130,9 +119,8 @@ jobs:
           readelf -d build/bin/sherpa-onnx
 
       - uses: actions/upload-artifact@v4
-        if: matrix.shared_lib == 'OFF' && matrix.build_type == 'Release'
         with:
-          name: release-static-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.shared_lib }}-${{ matrix.gcc_version }}
+          name: release-${{ matrix.build_type }}-${{ matrix.shared_lib }}
           path: build/bin/*
 
       - name: Test transducer kws
@@ -211,10 +199,17 @@ jobs:
 
       - name: Copy files
         shell: bash
+        if: matrix.build_type == 'Release'
         run: |
           SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
 
-          dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-x64
+          if [[ ${{ matrix.shared_lib }} == 'ON' ]]; then
+            suffix=shared
+          else
+            suffix=static
+          fi
+
+          dst=sherpa-onnx-${SHERPA_ONNX_VERSION}-linux-x64-$suffix
           mkdir $dst
 
           cp -a build/install/bin $dst/
@@ -225,16 +220,46 @@ jobs:
 
           tar cjvf ${dst}.tar.bz2 $dst
 
+      - name: Publish to huggingface
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && matrix.build_type == 'Release'
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "Fangjun Kuang"
+
+            GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
+
+            cd huggingface
+            git lfs pull
+            mkdir -p linux-x64
+
+            cp -v ../sherpa-onnx-*.tar.bz2 ./linux-x64
+
+            git status
+            git lfs track "*.bz2"
+
+            git add .
+
+            git commit -m "upload sherpa-onnx-${SHERPA_ONNX_VERSION}"
+
+            git push https://csukuangfj:[email protected]/csukuangfj/sherpa-onnx-libs main
+
       - name: Release pre-compiled binaries and libs for linux x64
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-20.04' && matrix.gcc_version == '7'
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.build_type == 'Release'
         uses: svenstaro/upload-release-action@v2
         with:
           file_glob: true
           overwrite: true
-          file: sherpa-onnx-*linux-x64.tar.bz2
+          file: sherpa-onnx-*.tar.bz2
 
       - uses: actions/upload-artifact@v4
         with:
-          name: tts-generated-test-files-${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.shared_lib }}-${{ matrix.gcc_version }}
+          name: tts-generated-test-files-${{ matrix.build_type }}-${{ matrix.shared_lib }}
           path: tts
 
diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml
@@ -225,7 +225,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml
@@ -154,7 +154,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml
@@ -157,7 +157,7 @@ jobs:
 
       # https://huggingface.co/docs/hub/spaces-github-actions
       - name: Publish to huggingface
-        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch')
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         uses: nick-fields/retry@v3

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-onnx)
 
-set(SHERPA_ONNX_VERSION "1.9.12")
+set(SHERPA_ONNX_VERSION "1.9.13")
 
 # Disable warning about
 #