From 07b86a2d9a0123311d1cea747fcbd96e4067daae Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Thu, 5 Oct 2023 13:57:36 +0900 Subject: [PATCH 1/3] =?UTF-8?q?cuda=E3=81=AE=E3=82=AD=E3=83=A3=E3=83=83?= =?UTF-8?q?=E3=82=B7=E3=83=A5=E3=82=92=E4=BD=9C=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 516 ++++++++++++++++++------------------ 1 file changed, 258 insertions(+), 258 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 629d3d125..5130cbb52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -368,261 +368,261 @@ jobs: run: | df -h - # Download VOICEVOX RESOURCE - - name: Prepare VOICEVOX RESOURCE cache - uses: actions/cache@v3 - id: voicevox-resource-cache - with: - key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - - name: Checkout VOICEVOX RESOURCE - if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v3 - with: - repository: VOICEVOX/voicevox_resource - ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} - path: download/resource - - # Merge VOICEVOX RESOURCE - - name: Merge VOICEVOX RESOURCE - shell: bash - env: - DOWNLOAD_RESOURCE_PATH: download/resource - run: bash build_util/process_voicevox_resource.bash - - # Download VOICEVOX Core - - name: Prepare VOICEVOX Core cache - uses: actions/cache@v3 - id: voicevox-core-cache - with: - key: ${{ matrix.os }}-voicevox-core-${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - path: download/core - - - name: Download VOICEVOX Core - if: steps.voicevox-core-cache.outputs.cache-hit != 'true' - shell: bash - env: - VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - run: | - curl -L "https://github.com/VOICEVOX/voicevox_core/releases/download/${{ env.VOICEVOX_CORE_VERSION }}/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip" > download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - # NOTE: Windows 版コアのみ PowerShell の Compress-Archive コマンドレットを用いて zip を作成している（デフォルト状態では zip コマンドが存在していないため）。 - # このコマンドはバージョンによっては作成した zip 内のパスの区切り文字がバックスラッシュになる。 (cf. https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/48) - # unzip コマンドはこのような zip ファイルを解凍できるものの、終了コード 1 を報告して CI が落ちてしまう。 - # 回避策として、unzip コマンドの代わりに 7z コマンドを用いて zip ファイルを解凍する。 - # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ - if [[ ${{ matrix.os }} == windows-* ]]; then - 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - elif [[ ${{ matrix.os }} == mac-* ]]; then - ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ - else - unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ - fi - mkdir -p download/core - mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core - rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} - rm download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - - - name: Copy VOICEVOX Core Models - shell: bash - run: | - cp -r download/core/model ./ - - - name: Generate licenses.json - shell: bash - run: | - OUTPUT_LICENSE_JSON_PATH=engine_manifest_assets/dependency_licenses.json \ - bash build_util/create_venv_and_generate_licenses.bash - # FIXME: VOICEVOX (editor) cannot build without licenses.json - cp engine_manifest_assets/dependency_licenses.json licenses.json - - - name: Show disk space (debug info) - shell: bash - run: | - df -h - - - name: Build run.py with PyInstaller - shell: bash - run: | - set -eux - - # Replace version & specify dynamic libraries - if [[ ${{ matrix.os }} == macos-* ]]; then - gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py - LIBCORE_PATH=download/core/libvoicevox_core.dylib - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib - else - sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py - if [[ ${{ matrix.os }} == windows-* ]]; then - LIBCORE_PATH=download/core/voicevox_core.dll - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll - else - LIBCORE_PATH=download/core/libvoicevox_core.so - LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so - fi - fi - - LIBCORE_PATH="$LIBCORE_PATH" \ - LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ - pyinstaller --noconfirm run.spec - - - name: Show disk space (debug info) - shell: bash - run: | - df -h - - - name: Create artifact directory with symlink - if: startsWith(matrix.os, 'windows-') - shell: bash - env: - PYTHON_SITE_PACKAGES_DIR: C:/hostedtoolcache/windows/python/${{ steps.setup-python.outputs.python-version }}/x64/lib/site-packages - # create symlink instead of copy (Git Bash) - # https://qiita.com/ucho/items/c5ea0beb8acf2f1e4772 - MSYS: winsymlinks:nativestrict - run: | - set -eux - - # Copy DLL dependencies - - if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then - # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run/ - - # CUDA - ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run/ - - # cuDNN - ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run/ - ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run/ - - # zlib - ln -sf "$(pwd)/download/zlib"/zlibwapi.dll dist/run/ - fi - - if [[ ${{ matrix.target }} == *-directml ]]; then - # DirectML - ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run/ - fi - - - name: Create symlink of CUDA dependencies - if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') - shell: bash - run: | - set -eux - - # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so - ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run/ - - # CUDA - ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run/ - ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run/ - - # cuDNN - ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run/ - ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run/ - - - name: Set @rpath to @executable_path - if: startsWith(matrix.os, 'macos-') - run: | - install_name_tool -add_rpath @executable_path/. dist/run/run - - - name: Code signing - if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') - shell: bash - run: | - bash build_util/codesign.bash "dist/run/run.exe" - env: - ESIGNERCKA_USERNAME: ${{ secrets.ESIGNERCKA_USERNAME }} - ESIGNERCKA_PASSWORD: ${{ secrets.ESIGNERCKA_PASSWORD }} - ESIGNERCKA_TOTP_SECRET: ${{ secrets.ESIGNERCKA_TOTP_SECRET }} - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.vars.outputs.artifact_name }} - path: dist/run/ - - upload-to-release: - if: needs.config.outputs.version != '' - needs: [config, build-all] - runs-on: ubuntu-latest - strategy: - matrix: - target: - - macos-x64 - - linux-cpu - - linux-nvidia - - windows-cpu - - windows-directml - - windows-nvidia - steps: - - name: declare variables - id: vars - shell: bash - run: | - echo "package_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version }}" >> $GITHUB_OUTPUT - : # this is exactly same as `steps.vars.outputs.artifact_name` in `build-all` job, - : # but since we cannot get the job outputs of matrix builds correctly, we need to redefine here. - echo "artifact_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version_or_latest }}-${{ github.sha }}" >> $GITHUB_OUTPUT - - - uses: actions/checkout@v3 - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - p7zip-full - - - name: Download and extract artifact - uses: actions/download-artifact@v3 - with: - name: ${{ steps.vars.outputs.artifact_name }} - path: ${{ matrix.target }}/ - - - name: Rearchive and split artifact - run: | - # Compress to artifact.7z.001, artifact.7z.002, ... - 7z -r -v1900m a "${{ steps.vars.outputs.package_name }}.7z" "${{ matrix.target }}/" - - # Compress to artifact.001.vvppp,artifact.002.vvppp, ... - (cd "${{ matrix.target }}" && zip -r - . > ../compressed.zip) - split -b 1900M --numeric-suffixes=1 -a 3 --additional-suffix .vvppp ./compressed.zip ./${{ steps.vars.outputs.package_name }}. - - # Rename to artifact.vvpp if there are only artifact.001.vvppp - if [ "$(ls ${{ steps.vars.outputs.package_name }}.*.vvppp | wc -l)" == 1 ]; then - mv ${{ steps.vars.outputs.package_name }}.001.vvppp ${{ steps.vars.outputs.package_name }}.vvpp - fi - - # Output splitted archive list - ls ${{ steps.vars.outputs.package_name }}.7z.* > archives_7z.txt - mv archives_7z.txt "${{ steps.vars.outputs.package_name }}.7z.txt" - ls ${{ steps.vars.outputs.package_name }}*.vvppp ${{ steps.vars.outputs.package_name }}.vvpp > archives_vvpp.txt || true - mv archives_vvpp.txt "${{ steps.vars.outputs.package_name }}.vvpp.txt" - - - name: Upload splitted archives to Release assets - uses: softprops/action-gh-release@v1 - with: - prerelease: ${{ github.event.inputs.prerelease }} - tag_name: ${{ needs.config.outputs.version }} - files: |- - ${{ steps.vars.outputs.package_name }}.7z.* - ${{ steps.vars.outputs.package_name }}.vvpp - ${{ steps.vars.outputs.package_name }}*.vvppp - ${{ steps.vars.outputs.package_name }}.vvpp.txt - target_commitish: ${{ github.sha }} - - run-release-test-workflow: - if: needs.config.outputs.version != '' - needs: [config, upload-to-release] - uses: ./.github/workflows/release-test.yml - with: - version: ${{ needs.config.outputs.version }} - repo_url: ${{ format('{0}/{1}', github.server_url, github.repository) }} # このリポジトリのURL + # # Download VOICEVOX RESOURCE + # - name: Prepare VOICEVOX RESOURCE cache + # uses: actions/cache@v3 + # id: voicevox-resource-cache + # with: + # key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} + # path: download/resource + + # - name: Checkout VOICEVOX RESOURCE + # if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' + # uses: actions/checkout@v3 + # with: + # repository: VOICEVOX/voicevox_resource + # ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} + # path: download/resource + + # # Merge VOICEVOX RESOURCE + # - name: Merge VOICEVOX RESOURCE + # shell: bash + # env: + # DOWNLOAD_RESOURCE_PATH: download/resource + # run: bash build_util/process_voicevox_resource.bash + + # # Download VOICEVOX Core + # - name: Prepare VOICEVOX Core cache + # uses: actions/cache@v3 + # id: voicevox-core-cache + # with: + # key: ${{ matrix.os }}-voicevox-core-${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} + # path: download/core + + # - name: Download VOICEVOX Core + # if: steps.voicevox-core-cache.outputs.cache-hit != 'true' + # shell: bash + # env: + # VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} + # run: | + # curl -L "https://github.com/VOICEVOX/voicevox_core/releases/download/${{ env.VOICEVOX_CORE_VERSION }}/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip" > download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + # # NOTE: Windows 版コアのみ PowerShell の Compress-Archive コマンドレットを用いて zip を作成している（デフォルト状態では zip コマンドが存在していないため）。 + # # このコマンドはバージョンによっては作成した zip 内のパスの区切り文字がバックスラッシュになる。 (cf. https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/48) + # # unzip コマンドはこのような zip ファイルを解凍できるものの、終了コード 1 を報告して CI が落ちてしまう。 + # # 回避策として、unzip コマンドの代わりに 7z コマンドを用いて zip ファイルを解凍する。 + # # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ + # if [[ ${{ matrix.os }} == windows-* ]]; then + # 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + # elif [[ ${{ matrix.os }} == mac-* ]]; then + # ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ + # else + # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ + # fi + # mkdir -p download/core + # mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core + # rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} + # rm download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + + # - name: Copy VOICEVOX Core Models + # shell: bash + # run: | + # cp -r download/core/model ./ + + # - name: Generate licenses.json + # shell: bash + # run: | + # OUTPUT_LICENSE_JSON_PATH=engine_manifest_assets/dependency_licenses.json \ + # bash build_util/create_venv_and_generate_licenses.bash + # # FIXME: VOICEVOX (editor) cannot build without licenses.json + # cp engine_manifest_assets/dependency_licenses.json licenses.json + + # - name: Show disk space (debug info) + # shell: bash + # run: | + # df -h + + # - name: Build run.py with PyInstaller + # shell: bash + # run: | + # set -eux + + # # Replace version & specify dynamic libraries + # if [[ ${{ matrix.os }} == macos-* ]]; then + # gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py + # LIBCORE_PATH=download/core/libvoicevox_core.dylib + # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib + # else + # sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py + # if [[ ${{ matrix.os }} == windows-* ]]; then + # LIBCORE_PATH=download/core/voicevox_core.dll + # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + # else + # LIBCORE_PATH=download/core/libvoicevox_core.so + # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so + # fi + # fi + + # LIBCORE_PATH="$LIBCORE_PATH" \ + # LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ + # pyinstaller --noconfirm run.spec + + # - name: Show disk space (debug info) + # shell: bash + # run: | + # df -h + + # - name: Create artifact directory with symlink + # if: startsWith(matrix.os, 'windows-') + # shell: bash + # env: + # PYTHON_SITE_PACKAGES_DIR: C:/hostedtoolcache/windows/python/${{ steps.setup-python.outputs.python-version }}/x64/lib/site-packages + # # create symlink instead of copy (Git Bash) + # # https://qiita.com/ucho/items/c5ea0beb8acf2f1e4772 + # MSYS: winsymlinks:nativestrict + # run: | + # set -eux + + # # Copy DLL dependencies + + # if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then + # # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + # ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run/ + + # # CUDA + # ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run/ + + # # cuDNN + # ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run/ + # ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run/ + + # # zlib + # ln -sf "$(pwd)/download/zlib"/zlibwapi.dll dist/run/ + # fi + + # if [[ ${{ matrix.target }} == *-directml ]]; then + # # DirectML + # ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run/ + # fi + + # - name: Create symlink of CUDA dependencies + # if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') + # shell: bash + # run: | + # set -eux + + # # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + # patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so + # ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run/ + + # # CUDA + # ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run/ + # ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run/ + + # # cuDNN + # ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run/ + # ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run/ + + # - name: Set @rpath to @executable_path + # if: startsWith(matrix.os, 'macos-') + # run: | + # install_name_tool -add_rpath @executable_path/. dist/run/run + + # - name: Code signing + # if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') + # shell: bash + # run: | + # bash build_util/codesign.bash "dist/run/run.exe" + # env: + # ESIGNERCKA_USERNAME: ${{ secrets.ESIGNERCKA_USERNAME }} + # ESIGNERCKA_PASSWORD: ${{ secrets.ESIGNERCKA_PASSWORD }} + # ESIGNERCKA_TOTP_SECRET: ${{ secrets.ESIGNERCKA_TOTP_SECRET }} + + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ steps.vars.outputs.artifact_name }} + # path: dist/run/ + + # upload-to-release: + # if: needs.config.outputs.version != '' + # needs: [config, build-all] + # runs-on: ubuntu-latest + # strategy: + # matrix: + # target: + # - macos-x64 + # - linux-cpu + # - linux-nvidia + # - windows-cpu + # - windows-directml + # - windows-nvidia + # steps: + # - name: declare variables + # id: vars + # shell: bash + # run: | + # echo "package_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version }}" >> $GITHUB_OUTPUT + # : # this is exactly same as `steps.vars.outputs.artifact_name` in `build-all` job, + # : # but since we cannot get the job outputs of matrix builds correctly, we need to redefine here. + # echo "artifact_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version_or_latest }}-${{ github.sha }}" >> $GITHUB_OUTPUT + + # - uses: actions/checkout@v3 + + # - name: Install dependencies + # run: | + # sudo apt-get update + # sudo apt-get install -y \ + # p7zip-full + + # - name: Download and extract artifact + # uses: actions/download-artifact@v3 + # with: + # name: ${{ steps.vars.outputs.artifact_name }} + # path: ${{ matrix.target }}/ + + # - name: Rearchive and split artifact + # run: | + # # Compress to artifact.7z.001, artifact.7z.002, ... + # 7z -r -v1900m a "${{ steps.vars.outputs.package_name }}.7z" "${{ matrix.target }}/" + + # # Compress to artifact.001.vvppp,artifact.002.vvppp, ... + # (cd "${{ matrix.target }}" && zip -r - . > ../compressed.zip) + # split -b 1900M --numeric-suffixes=1 -a 3 --additional-suffix .vvppp ./compressed.zip ./${{ steps.vars.outputs.package_name }}. + + # # Rename to artifact.vvpp if there are only artifact.001.vvppp + # if [ "$(ls ${{ steps.vars.outputs.package_name }}.*.vvppp | wc -l)" == 1 ]; then + # mv ${{ steps.vars.outputs.package_name }}.001.vvppp ${{ steps.vars.outputs.package_name }}.vvpp + # fi + + # # Output splitted archive list + # ls ${{ steps.vars.outputs.package_name }}.7z.* > archives_7z.txt + # mv archives_7z.txt "${{ steps.vars.outputs.package_name }}.7z.txt" + # ls ${{ steps.vars.outputs.package_name }}*.vvppp ${{ steps.vars.outputs.package_name }}.vvpp > archives_vvpp.txt || true + # mv archives_vvpp.txt "${{ steps.vars.outputs.package_name }}.vvpp.txt" + + # - name: Upload splitted archives to Release assets + # uses: softprops/action-gh-release@v1 + # with: + # prerelease: ${{ github.event.inputs.prerelease }} + # tag_name: ${{ needs.config.outputs.version }} + # files: |- + # ${{ steps.vars.outputs.package_name }}.7z.* + # ${{ steps.vars.outputs.package_name }}.vvpp + # ${{ steps.vars.outputs.package_name }}*.vvppp + # ${{ steps.vars.outputs.package_name }}.vvpp.txt + # target_commitish: ${{ github.sha }} + + # run-release-test-workflow: + # if: needs.config.outputs.version != '' + # needs: [config, upload-to-release] + # uses: ./.github/workflows/release-test.yml + # with: + # version: ${{ needs.config.outputs.version }} + # repo_url: ${{ format('{0}/{1}', github.server_url, github.repository) }} # このリポジトリのURL From 6612c1ec01c53d0b352b6a80c4ce7c0ec1d20070 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Thu, 5 Oct 2023 15:39:53 +0900 Subject: [PATCH 2/3] =?UTF-8?q?#=20Download=20VOICEVOX=20RESOURCE=E3=81=BE?= =?UTF-8?q?=E3=81=A7=E3=81=AE=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E3=82=B3?= =?UTF-8?q?=E3=83=94=E3=83=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 846 +++++++++++++++++++++++++----------- 1 file changed, 587 insertions(+), 259 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5130cbb52..d00da8fba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,7 @@ jobs: : # release タグ名, または workflow_dispatch でのバージョン名, または 'latest' echo "version_or_latest=${{ github.event.release.tag_name || github.event.inputs.version || 'latest' }}" >> $GITHUB_OUTPUT - build-all: + hotfix-cache-only: needs: [config] environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' || '' }} # コード署名用のenvironment strategy: @@ -368,261 +368,589 @@ jobs: run: | df -h - # # Download VOICEVOX RESOURCE - # - name: Prepare VOICEVOX RESOURCE cache - # uses: actions/cache@v3 - # id: voicevox-resource-cache - # with: - # key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} - # path: download/resource - - # - name: Checkout VOICEVOX RESOURCE - # if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' - # uses: actions/checkout@v3 - # with: - # repository: VOICEVOX/voicevox_resource - # ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} - # path: download/resource - - # # Merge VOICEVOX RESOURCE - # - name: Merge VOICEVOX RESOURCE - # shell: bash - # env: - # DOWNLOAD_RESOURCE_PATH: download/resource - # run: bash build_util/process_voicevox_resource.bash - - # # Download VOICEVOX Core - # - name: Prepare VOICEVOX Core cache - # uses: actions/cache@v3 - # id: voicevox-core-cache - # with: - # key: ${{ matrix.os }}-voicevox-core-${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - # path: download/core - - # - name: Download VOICEVOX Core - # if: steps.voicevox-core-cache.outputs.cache-hit != 'true' - # shell: bash - # env: - # VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} - # run: | - # curl -L "https://github.com/VOICEVOX/voicevox_core/releases/download/${{ env.VOICEVOX_CORE_VERSION }}/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip" > download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - # # NOTE: Windows 版コアのみ PowerShell の Compress-Archive コマンドレットを用いて zip を作成している（デフォルト状態では zip コマンドが存在していないため）。 - # # このコマンドはバージョンによっては作成した zip 内のパスの区切り文字がバックスラッシュになる。 (cf. https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/48) - # # unzip コマンドはこのような zip ファイルを解凍できるものの、終了コード 1 を報告して CI が落ちてしまう。 - # # 回避策として、unzip コマンドの代わりに 7z コマンドを用いて zip ファイルを解凍する。 - # # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ - # if [[ ${{ matrix.os }} == windows-* ]]; then - # 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - # elif [[ ${{ matrix.os }} == mac-* ]]; then - # ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ - # else - # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ - # fi - # mkdir -p download/core - # mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core - # rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} - # rm download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip - - # - name: Copy VOICEVOX Core Models - # shell: bash - # run: | - # cp -r download/core/model ./ - - # - name: Generate licenses.json - # shell: bash - # run: | - # OUTPUT_LICENSE_JSON_PATH=engine_manifest_assets/dependency_licenses.json \ - # bash build_util/create_venv_and_generate_licenses.bash - # # FIXME: VOICEVOX (editor) cannot build without licenses.json - # cp engine_manifest_assets/dependency_licenses.json licenses.json - - # - name: Show disk space (debug info) - # shell: bash - # run: | - # df -h - - # - name: Build run.py with PyInstaller - # shell: bash - # run: | - # set -eux - - # # Replace version & specify dynamic libraries - # if [[ ${{ matrix.os }} == macos-* ]]; then - # gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py - # LIBCORE_PATH=download/core/libvoicevox_core.dylib - # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib - # else - # sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py - # if [[ ${{ matrix.os }} == windows-* ]]; then - # LIBCORE_PATH=download/core/voicevox_core.dll - # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll - # else - # LIBCORE_PATH=download/core/libvoicevox_core.so - # LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so - # fi - # fi - - # LIBCORE_PATH="$LIBCORE_PATH" \ - # LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ - # pyinstaller --noconfirm run.spec - - # - name: Show disk space (debug info) - # shell: bash - # run: | - # df -h - - # - name: Create artifact directory with symlink - # if: startsWith(matrix.os, 'windows-') - # shell: bash - # env: - # PYTHON_SITE_PACKAGES_DIR: C:/hostedtoolcache/windows/python/${{ steps.setup-python.outputs.python-version }}/x64/lib/site-packages - # # create symlink instead of copy (Git Bash) - # # https://qiita.com/ucho/items/c5ea0beb8acf2f1e4772 - # MSYS: winsymlinks:nativestrict - # run: | - # set -eux - - # # Copy DLL dependencies - - # if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then - # # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - # ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run/ - - # # CUDA - # ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run/ - - # # cuDNN - # ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run/ - # ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run/ - - # # zlib - # ln -sf "$(pwd)/download/zlib"/zlibwapi.dll dist/run/ - # fi - - # if [[ ${{ matrix.target }} == *-directml ]]; then - # # DirectML - # ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run/ - # fi - - # - name: Create symlink of CUDA dependencies - # if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') - # shell: bash - # run: | - # set -eux - - # # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) - # patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so - # ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run/ - - # # CUDA - # ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run/ - # ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run/ - - # # cuDNN - # ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run/ - # ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run/ - - # - name: Set @rpath to @executable_path - # if: startsWith(matrix.os, 'macos-') - # run: | - # install_name_tool -add_rpath @executable_path/. dist/run/run - - # - name: Code signing - # if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') - # shell: bash - # run: | - # bash build_util/codesign.bash "dist/run/run.exe" - # env: - # ESIGNERCKA_USERNAME: ${{ secrets.ESIGNERCKA_USERNAME }} - # ESIGNERCKA_PASSWORD: ${{ secrets.ESIGNERCKA_PASSWORD }} - # ESIGNERCKA_TOTP_SECRET: ${{ secrets.ESIGNERCKA_TOTP_SECRET }} - - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ steps.vars.outputs.artifact_name }} - # path: dist/run/ - - # upload-to-release: - # if: needs.config.outputs.version != '' - # needs: [config, build-all] - # runs-on: ubuntu-latest - # strategy: - # matrix: - # target: - # - macos-x64 - # - linux-cpu - # - linux-nvidia - # - windows-cpu - # - windows-directml - # - windows-nvidia - # steps: - # - name: declare variables - # id: vars - # shell: bash - # run: | - # echo "package_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version }}" >> $GITHUB_OUTPUT - # : # this is exactly same as `steps.vars.outputs.artifact_name` in `build-all` job, - # : # but since we cannot get the job outputs of matrix builds correctly, we need to redefine here. - # echo "artifact_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version_or_latest }}-${{ github.sha }}" >> $GITHUB_OUTPUT - - # - uses: actions/checkout@v3 - - # - name: Install dependencies - # run: | - # sudo apt-get update - # sudo apt-get install -y \ - # p7zip-full - - # - name: Download and extract artifact - # uses: actions/download-artifact@v3 - # with: - # name: ${{ steps.vars.outputs.artifact_name }} - # path: ${{ matrix.target }}/ - - # - name: Rearchive and split artifact - # run: | - # # Compress to artifact.7z.001, artifact.7z.002, ... - # 7z -r -v1900m a "${{ steps.vars.outputs.package_name }}.7z" "${{ matrix.target }}/" - - # # Compress to artifact.001.vvppp,artifact.002.vvppp, ... - # (cd "${{ matrix.target }}" && zip -r - . > ../compressed.zip) - # split -b 1900M --numeric-suffixes=1 -a 3 --additional-suffix .vvppp ./compressed.zip ./${{ steps.vars.outputs.package_name }}. - - # # Rename to artifact.vvpp if there are only artifact.001.vvppp - # if [ "$(ls ${{ steps.vars.outputs.package_name }}.*.vvppp | wc -l)" == 1 ]; then - # mv ${{ steps.vars.outputs.package_name }}.001.vvppp ${{ steps.vars.outputs.package_name }}.vvpp - # fi - - # # Output splitted archive list - # ls ${{ steps.vars.outputs.package_name }}.7z.* > archives_7z.txt - # mv archives_7z.txt "${{ steps.vars.outputs.package_name }}.7z.txt" - # ls ${{ steps.vars.outputs.package_name }}*.vvppp ${{ steps.vars.outputs.package_name }}.vvpp > archives_vvpp.txt || true - # mv archives_vvpp.txt "${{ steps.vars.outputs.package_name }}.vvpp.txt" - - # - name: Upload splitted archives to Release assets - # uses: softprops/action-gh-release@v1 - # with: - # prerelease: ${{ github.event.inputs.prerelease }} - # tag_name: ${{ needs.config.outputs.version }} - # files: |- - # ${{ steps.vars.outputs.package_name }}.7z.* - # ${{ steps.vars.outputs.package_name }}.vvpp - # ${{ steps.vars.outputs.package_name }}*.vvppp - # ${{ steps.vars.outputs.package_name }}.vvpp.txt - # target_commitish: ${{ github.sha }} - - # run-release-test-workflow: - # if: needs.config.outputs.version != '' - # needs: [config, upload-to-release] - # uses: ./.github/workflows/release-test.yml - # with: - # version: ${{ needs.config.outputs.version }} - # repo_url: ${{ format('{0}/{1}', github.server_url, github.repository) }} # このリポジトリのURL + build-all: + needs: [config, hotfix-cache-only] + environment: ${{ github.event.inputs.code_signing == 'true' && 'code_signing' || '' }} # コード署名用のenvironment + strategy: + matrix: + include: + # Windows CPU + - os: windows-2019 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-windows-x64-cpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-win-x64-1.13.1.zip + target: windows-cpu + # Windows DirectML + - os: windows-2019 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-windows-x64-directml + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/Microsoft.ML.OnnxRuntime.DirectML.1.13.1.zip + directml_url: https://www.nuget.org/api/v2/package/Microsoft.AI.DirectML/1.10.0 + target: windows-directml + # Windows NVIDIA GPU + - os: windows-2019 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-windows-x64-cuda + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-win-x64-gpu-1.13.1.zip + cuda_version: "11.6.2" + cudnn_url: https://developer.download.nvidia.com/compute/redist/cudnn/v8.4.1/local_installers/11.6/cudnn-windows-x86_64-8.4.1.50_cuda11.6-archive.zip + zlib_url: http://www.winimage.com/zLibDll/zlib123dllx64.zip + target: windows-nvidia + # Mac CPU (x64 arch only) + - os: macos-11 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-osx-x64-cpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-osx-x86_64-1.13.1.tgz + target: macos-x64 + # Linux CPU + - os: ubuntu-20.04 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-linux-x64-cpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-linux-x64-1.13.1.tgz + target: linux-cpu + # Linux NVIDIA GPU + - os: ubuntu-20.04 + architecture: "x64" + voicevox_core_asset_prefix: voicevox_core-linux-x64-gpu + onnxruntime_url: https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-linux-x64-gpu-1.13.1.tgz + cuda_version: "11.6.2" + cudnn_url: https://developer.download.nvidia.com/compute/redist/cudnn/v8.4.1/local_installers/11.6/cudnn-linux-x86_64-8.4.1.50_cuda11.6-archive.tar.xz + target: linux-nvidia + + runs-on: ${{ matrix.os }} + + steps: + - name: declare variables + id: vars + shell: bash + run: | + echo "artifact_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version_or_latest }}-${{ github.sha }}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v3 + + - name: Show disk space (debug info) + shell: bash + run: | + df -h + + # NOTE: The default sed of macOS is BSD sed. + # There is a difference in specification between BSD sed and GNU sed, + # so you need to install GNU sed. + - name: Install GNU sed on macOS + if: startsWith(matrix.os, 'macos-') + shell: bash + run: brew install gnu-sed + + # ONNX Runtime providersとCUDA周りをリンクするために使う + - name: Install patchelf + if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') + run: | + sudo apt-get update + sudo apt-get install -y patchelf + + # Download CUDA + - name: Prepare CUDA DLL cache + if: matrix.cuda_version != '' + uses: actions/cache@v3 + id: cuda-dll-cache + with: + # update this key when ONNX Runtime CUDA dependency changed + key: ${{ matrix.os }}-cuda-dll-${{ matrix.cuda_version }}-v1 + path: download/cuda + + - name: Setup CUDA + if: matrix.cuda_version != '' && steps.cuda-dll-cache.outputs.cache-hit != 'true' + uses: Jimver/cuda-toolkit@v0.2.8 + id: cuda-toolkit + with: + method: network + cuda: ${{ matrix.cuda_version }} + + - name: Extract CUDA Dynamic Libraries + if: matrix.cuda_version != '' && steps.cuda-dll-cache.outputs.cache-hit != 'true' + shell: bash + run: | + set -eux + + CUDA_ROOT=$( echo "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" | tr '\\' '/' ) + + mkdir -p download/cuda/bin + if [[ ${{ matrix.os }} == windows-* ]]; then + mv "${CUDA_ROOT}/bin/"*.dll download/cuda/bin/ + + rm -rf "${CUDA_ROOT}" + else + cp "${CUDA_ROOT}/lib64/"libcublas.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcublasLt.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcudart.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcufft.so.* download/cuda/bin/ + cp "${CUDA_ROOT}/lib64/"libcurand.so.* download/cuda/bin/ + + # remove unneed full version libraries + rm -f download/cuda/bin/libcublas.so.*.* + rm -f download/cuda/bin/libcublasLt.so.*.* + rm -f download/cuda/bin/libcufft.so.*.* + rm -f download/cuda/bin/libcurand.so.*.* + rm -f download/cuda/bin/libcudart.so.*.*.* + fi + + - name: Show disk space (debug info) + if: matrix.cuda_version != '' + shell: bash + run: | + df -h + + # Download cuDNN + - name: Export cuDNN url to calc hash + if: matrix.cudnn_url != '' + shell: bash + run: echo "${{ matrix.cudnn_url }}" > download/cudnn_url.txt + + - name: Prepare cuDNN cache + if: matrix.cudnn_url != '' + uses: actions/cache@v3 + id: cudnn-dll-cache + with: + # update this key when ONNX Runtime cuDNN dependency changed + key: ${{ matrix.os }}-cudnn-dll-${{ hashFiles('download/cudnn_url.txt') }}-v1 + path: download/cudnn + + - name: Download and extract cuDNN Dynamic Libraries + if: matrix.cudnn_url != '' && steps.cudnn-dll-cache.outputs.cache-hit != 'true' + shell: bash + run: | + set -eux + + if [[ ${{ matrix.os }} == windows-* ]]; then + curl -L "${{ matrix.cudnn_url }}" > download/cudnn.zip + + unzip download/cudnn.zip cudnn-*/bin/*.dll -d download/cudnn_tmp + + mkdir -p download/cudnn/bin + mv download/cudnn_tmp/cudnn-*/bin/*.dll download/cudnn/bin/ + rm -rf download/cudnn_tmp + + rm download/cudnn.zip + else + curl -L "${{ matrix.cudnn_url }}" > download/cudnn.tar.xz + + tar -Jxf download/cudnn.tar.xz -C download/ + + mkdir -p download/cudnn/bin + cp download/cudnn-*/lib/libcudnn.so.* download/cudnn/bin/ + cp download/cudnn-*/lib/libcudnn_*_infer.so.* download/cudnn/bin/ + + # remove unneed full version libraries + rm -f download/cudnn/bin/libcudnn.so.*.* + rm -f download/cudnn/bin/libcudnn_*_infer.so.*.* + + rm download/cudnn.tar.xz + fi + + # Donwload zlib + - name: Export zlib url to calc hash + if: matrix.zlib_url != '' + shell: bash + run: echo "${{ matrix.zlib_url }}" >> download/zlib_url.txt + + - name: Cache zlib + if: matrix.zlib_url != '' + uses: actions/cache@v3 + id: zlib-cache + with: + key: zlib-cache-v1-${{ hashFiles('download/zlib_url.txt') }} + path: download/zlib + + - name: Download zlib + if: steps.zlib-cache.outputs.cache-hit != 'true' && matrix.zlib_url != '' + shell: bash + run: | + curl -L "${{ matrix.zlib_url }}" -o download/zlib.zip + mkdir -p download/zlib + + # extract only dlls + unzip download/zlib.zip dll_${{ matrix.architecture }}/zlibwapi.dll -d download/zlib + rm download/zlib.zip + mv download/zlib/dll_${{ matrix.architecture }}/zlibwapi.dll download/zlib/zlibwapi.dll + rm -r download/zlib/dll_${{ matrix.architecture }} + + - name: Show disk space (debug info) + if: matrix.cudnn_url != '' + shell: bash + run: | + df -h + + - name: Setup MSVC + if: startsWith(matrix.os, 'windows-') + uses: ilammy/msvc-dev-cmd@v1 + + # Python install path of windows: C:/hostedtoolcache/windows/Python + - name: Setup Python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + architecture: ${{ matrix.architecture }} + cache: pip + + - name: Install Python dependencies + shell: bash + run: | + python -m pip install -r requirements-dev.txt + + if [ "$RUNNER_OS" = Windows ]; then + # Modify PyInstaller to enable NvOptimusEnablement and AmdPowerXpressRequestHighPerformance + ./build_util/modify_pyinstaller.bash + fi + + # Download pyopenjtalk dictionary + # try 5 times, sleep 5 seconds before retry + for i in $(seq 5); do + EXIT_CODE=0 + python3 -c "import pyopenjtalk; pyopenjtalk._lazy_init()" || EXIT_CODE=$? + + if [ "$EXIT_CODE" = "0" ]; then + break + fi + + sleep 5 + done + + if [ "$EXIT_CODE" != "0" ]; then + exit "$EXIT_CODE" + fi + + - name: Create download directory + shell: bash + run: mkdir -p download/ + + # Donwload DirectML + - name: Export DirectML url to calc hash + if: endswith(matrix.target, '-directml') + shell: bash + run: echo "${{ matrix.directml_url }}" >> download/directml_url.txt + + - name: Cache DirectML + if: endswith(matrix.target, '-directml') + uses: actions/cache@v3 + id: directml-cache + with: + key: directml-cache-v1-${{ hashFiles('download/directml_url.txt') }} + path: download/directml + + - name: Download DirectML + if: steps.directml-cache.outputs.cache-hit != 'true' && endswith(matrix.target, '-directml') + shell: bash + run: | + curl -L "${{ matrix.directml_url }}" -o download/directml.zip + mkdir -p download/directml + + # extract only dlls + unzip download/directml.zip bin/${{ matrix.architecture }}-win/DirectML.dll -d download/directml + rm download/directml.zip + mv download/directml/bin/${{ matrix.architecture }}-win/DirectML.dll download/directml/DirectML.dll + rm -r download/directml/bin + + # Download ONNX Runtime + - name: Export ONNX Runtime url to calc hash + shell: bash + run: echo "${{ matrix.onnxruntime_url }}" > download/onnxruntime_url.txt + + - name: Prepare ONNX Runtime cache + uses: actions/cache@v3 + id: onnxruntime-cache + with: + key: ${{ matrix.os }}-onnxruntime-${{ hashFiles('download/onnxruntime_url.txt') }}-v1 + path: download/onnxruntime + + - name: Download ONNX Runtime (Windows) + if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'windows-') + shell: bash + run: | + curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.zip + + # extract only dlls + if [[ ${{ matrix.target }} != *-directml ]]; then + unzip download/onnxruntime.zip onnxruntime-*/lib/*.dll -d download/ + mv download/onnxruntime-* download/onnxruntime + else + mkdir -p download/onnxruntime/lib + unzip download/onnxruntime.zip runtimes/win-${{ matrix.architecture }}/native/*.dll -d download/onnxruntime + mv download/onnxruntime/runtimes/win-${{ matrix.architecture }}/native/*.dll download/onnxruntime/lib/ + rm -r download/onnxruntime/runtimes + fi + + rm download/onnxruntime.zip + + - name: Download ONNX Runtime (Mac/Linux) + if: steps.onnxruntime-cache.outputs.cache-hit != 'true' && startsWith(matrix.os, 'windows-') != true + shell: bash + run: | + curl -L "${{ matrix.onnxruntime_url }}" > download/onnxruntime.tgz + mkdir -p download/onnxruntime + tar xf "download/onnxruntime.tgz" -C "download/onnxruntime" --strip-components 1 + rm download/onnxruntime.tgz + + - name: Show disk space (debug info) + shell: bash + run: | + df -h + + # Download VOICEVOX RESOURCE + - name: Prepare VOICEVOX RESOURCE cache + uses: actions/cache@v3 + id: voicevox-resource-cache + with: + key: voicevox-resource-${{ env.VOICEVOX_RESOURCE_VERSION }} + path: download/resource + + - name: Checkout VOICEVOX RESOURCE + if: steps.voicevox-resource-cache.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: VOICEVOX/voicevox_resource + ref: ${{ env.VOICEVOX_RESOURCE_VERSION }} + path: download/resource + + # Merge VOICEVOX RESOURCE + - name: Merge VOICEVOX RESOURCE + shell: bash + env: + DOWNLOAD_RESOURCE_PATH: download/resource + run: bash build_util/process_voicevox_resource.bash + + # Download VOICEVOX Core + - name: Prepare VOICEVOX Core cache + uses: actions/cache@v3 + id: voicevox-core-cache + with: + key: ${{ matrix.os }}-voicevox-core-${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} + path: download/core + + - name: Download VOICEVOX Core + if: steps.voicevox-core-cache.outputs.cache-hit != 'true' + shell: bash + env: + VOICEVOX_CORE_ASSET_NAME: ${{ matrix.voicevox_core_asset_prefix }}-${{ env.VOICEVOX_CORE_VERSION }} + run: | + curl -L "https://github.com/VOICEVOX/voicevox_core/releases/download/${{ env.VOICEVOX_CORE_VERSION }}/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip" > download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + # NOTE: Windows 版コアのみ PowerShell の Compress-Archive コマンドレットを用いて zip を作成している（デフォルト状態では zip コマンドが存在していないため）。 + # このコマンドはバージョンによっては作成した zip 内のパスの区切り文字がバックスラッシュになる。 (cf. https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/48) + # unzip コマンドはこのような zip ファイルを解凍できるものの、終了コード 1 を報告して CI が落ちてしまう。 + # 回避策として、unzip コマンドの代わりに 7z コマンドを用いて zip ファイルを解凍する。 + # unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ + if [[ ${{ matrix.os }} == windows-* ]]; then + 7z x -o"download" download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + elif [[ ${{ matrix.os }} == mac-* ]]; then + ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ + else + unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ + fi + mkdir -p download/core + mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core + rm -rf download/${{ env.VOICEVOX_CORE_ASSET_NAME }} + rm download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip + + - name: Copy VOICEVOX Core Models + shell: bash + run: | + cp -r download/core/model ./ + + - name: Generate licenses.json + shell: bash + run: | + OUTPUT_LICENSE_JSON_PATH=engine_manifest_assets/dependency_licenses.json \ + bash build_util/create_venv_and_generate_licenses.bash + # FIXME: VOICEVOX (editor) cannot build without licenses.json + cp engine_manifest_assets/dependency_licenses.json licenses.json + + - name: Show disk space (debug info) + shell: bash + run: | + df -h + + - name: Build run.py with PyInstaller + shell: bash + run: | + set -eux + + # Replace version & specify dynamic libraries + if [[ ${{ matrix.os }} == macos-* ]]; then + gsed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py + LIBCORE_PATH=download/core/libvoicevox_core.dylib + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.dylib + else + sed -i "s/__version__ = \"latest\"/__version__ = \"${{ needs.config.outputs.version_or_latest }}\"/" voicevox_engine/__init__.py + if [[ ${{ matrix.os }} == windows-* ]]; then + LIBCORE_PATH=download/core/voicevox_core.dll + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/onnxruntime.dll + else + LIBCORE_PATH=download/core/libvoicevox_core.so + LIBONNXRUNTIME_PATH=download/onnxruntime/lib/libonnxruntime.so + fi + fi + + LIBCORE_PATH="$LIBCORE_PATH" \ + LIBONNXRUNTIME_PATH="$LIBONNXRUNTIME_PATH" \ + pyinstaller --noconfirm run.spec + + - name: Show disk space (debug info) + shell: bash + run: | + df -h + + - name: Create artifact directory with symlink + if: startsWith(matrix.os, 'windows-') + shell: bash + env: + PYTHON_SITE_PACKAGES_DIR: C:/hostedtoolcache/windows/python/${{ steps.setup-python.outputs.python-version }}/x64/lib/site-packages + # create symlink instead of copy (Git Bash) + # https://qiita.com/ucho/items/c5ea0beb8acf2f1e4772 + MSYS: winsymlinks:nativestrict + run: | + set -eux + + # Copy DLL dependencies + + if [ -f "download/onnxruntime/lib/onnxruntime_providers_cuda.dll" ]; then + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + ln -sf "$(pwd)/download/onnxruntime/lib"/onnxruntime_*.dll dist/run/ + + # CUDA + ln -sf "$(pwd)/download/cuda/bin"/cublas64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cublasLt64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cudart64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/cufft64_*.dll dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/curand64_*.dll dist/run/ + + # cuDNN + ln -sf "$(pwd)/download/cudnn/bin"/cudnn64_*.dll dist/run/ + ln -sf "$(pwd)/download/cudnn/bin"/cudnn_*_infer64*.dll dist/run/ + + # zlib + ln -sf "$(pwd)/download/zlib"/zlibwapi.dll dist/run/ + fi + + if [[ ${{ matrix.target }} == *-directml ]]; then + # DirectML + ln -sf "$(pwd)/download/directml"/DirectML.dll dist/run/ + fi + + - name: Create symlink of CUDA dependencies + if: startsWith(matrix.os, 'ubuntu-') && endsWith(matrix.target, 'nvidia') + shell: bash + run: | + set -eux + + # ONNX Runtime providers (PyInstaller does not copy dynamic loaded libraries) + patchelf --set-rpath '$ORIGIN' "$(pwd)/download/onnxruntime/lib"/libonnxruntime_providers_*.so + ln -sf "$(pwd)/download/onnxruntime/lib"/libonnxruntime_*.so dist/run/ + + # CUDA + ln -sf "$(pwd)/download/cuda/bin"/libcublas.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcublasLt.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcudart.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcufft.so.* dist/run/ + ln -sf "$(pwd)/download/cuda/bin"/libcurand.so.* dist/run/ + + # cuDNN + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn.so.* dist/run/ + ln -sf "$(pwd)/download/cudnn/bin"/libcudnn_*_infer.so.* dist/run/ + + - name: Set @rpath to @executable_path + if: startsWith(matrix.os, 'macos-') + run: | + install_name_tool -add_rpath @executable_path/. dist/run/run + + - name: Code signing + if: github.event.inputs.code_signing == 'true' && startsWith(matrix.os, 'windows-') + shell: bash + run: | + bash build_util/codesign.bash "dist/run/run.exe" + env: + ESIGNERCKA_USERNAME: ${{ secrets.ESIGNERCKA_USERNAME }} + ESIGNERCKA_PASSWORD: ${{ secrets.ESIGNERCKA_PASSWORD }} + ESIGNERCKA_TOTP_SECRET: ${{ secrets.ESIGNERCKA_TOTP_SECRET }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.vars.outputs.artifact_name }} + path: dist/run/ + + upload-to-release: + if: needs.config.outputs.version != '' + needs: [config, build-all] + runs-on: ubuntu-latest + strategy: + matrix: + target: + - macos-x64 + - linux-cpu + - linux-nvidia + - windows-cpu + - windows-directml + - windows-nvidia + steps: + - name: declare variables + id: vars + shell: bash + run: | + echo "package_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version }}" >> $GITHUB_OUTPUT + : # this is exactly same as `steps.vars.outputs.artifact_name` in `build-all` job, + : # but since we cannot get the job outputs of matrix builds correctly, we need to redefine here. + echo "artifact_name=voicevox_engine-${{ matrix.target }}-${{ needs.config.outputs.version_or_latest }}-${{ github.sha }}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v3 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + p7zip-full + + - name: Download and extract artifact + uses: actions/download-artifact@v3 + with: + name: ${{ steps.vars.outputs.artifact_name }} + path: ${{ matrix.target }}/ + + - name: Rearchive and split artifact + run: | + # Compress to artifact.7z.001, artifact.7z.002, ... + 7z -r -v1900m a "${{ steps.vars.outputs.package_name }}.7z" "${{ matrix.target }}/" + + # Compress to artifact.001.vvppp,artifact.002.vvppp, ... + (cd "${{ matrix.target }}" && zip -r - . > ../compressed.zip) + split -b 1900M --numeric-suffixes=1 -a 3 --additional-suffix .vvppp ./compressed.zip ./${{ steps.vars.outputs.package_name }}. + + # Rename to artifact.vvpp if there are only artifact.001.vvppp + if [ "$(ls ${{ steps.vars.outputs.package_name }}.*.vvppp | wc -l)" == 1 ]; then + mv ${{ steps.vars.outputs.package_name }}.001.vvppp ${{ steps.vars.outputs.package_name }}.vvpp + fi + + # Output splitted archive list + ls ${{ steps.vars.outputs.package_name }}.7z.* > archives_7z.txt + mv archives_7z.txt "${{ steps.vars.outputs.package_name }}.7z.txt" + ls ${{ steps.vars.outputs.package_name }}*.vvppp ${{ steps.vars.outputs.package_name }}.vvpp > archives_vvpp.txt || true + mv archives_vvpp.txt "${{ steps.vars.outputs.package_name }}.vvpp.txt" + + - name: Upload splitted archives to Release assets + uses: softprops/action-gh-release@v1 + with: + prerelease: ${{ github.event.inputs.prerelease }} + tag_name: ${{ needs.config.outputs.version }} + files: |- + ${{ steps.vars.outputs.package_name }}.7z.* + ${{ steps.vars.outputs.package_name }}.vvpp + ${{ steps.vars.outputs.package_name }}*.vvppp + ${{ steps.vars.outputs.package_name }}.vvpp.txt + target_commitish: ${{ github.sha }} + + run-release-test-workflow: + if: needs.config.outputs.version != '' + needs: [config, upload-to-release] + uses: ./.github/workflows/release-test.yml + with: + version: ${{ needs.config.outputs.version }} + repo_url: ${{ format('{0}/{1}', github.server_url, github.repository) }} # このリポジトリのURL From d347aaf63de08caf9001c930cb45899c3ec1fb76 Mon Sep 17 00:00:00 2001 From: Hiroshiba Date: Thu, 5 Oct 2023 15:41:10 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E3=81=AA=E3=82=93=E3=81=8B=E3=82=B9?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B9=E3=81=8C=E3=81=82=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d00da8fba..bcffc4d51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -744,7 +744,7 @@ jobs: elif [[ ${{ matrix.os }} == mac-* ]]; then ditto -x -k --sequesterRsrc --rsrc download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip download/ else - unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ + unzip download/${{ env.VOICEVOX_CORE_ASSET_NAME }}.zip -d download/ fi mkdir -p download/core mv download/${{ env.VOICEVOX_CORE_ASSET_NAME }}/* download/core