From a000c66d9710091435d2936e8412b6557c3c588b Mon Sep 17 00:00:00 2001 From: aoirint Date: Sat, 14 Dec 2024 08:04:43 +0900 Subject: [PATCH] =?UTF-8?q?Docker=E3=83=93=E3=83=AB=E3=83=89CI:=20?= =?UTF-8?q?=E3=82=A4=E3=83=A1=E3=83=BC=E3=82=B8=E3=82=BF=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E5=88=A5=E5=90=8D=E3=82=92=E5=88=A5=E3=80=85=E3=81=AB=E3=83=93?= =?UTF-8?q?=E3=83=AB=E3=83=89=E3=81=97=E3=81=AA=E3=81=84=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B=20(#1498)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Docker: イメージタグの別名を別々にビルドしないようにする * create tools/generate_docker_image_names.bash * comment: quote shell args * comment: improve examples * comment: improve examples * comment: improve examples * comment: 冗長な例を削除 * -eq -> = * fix SC2086: double quote $GITHUB_OUTPUT * fix wrong step output reference * remove shell: bash * コンフリクト解消時にUbuntu 22.04のincludeが増殖していたのを削除 * print usage to stderr * add matrix var descriptions * add comment * fix comment * update comment * fix buildcache image name: add repository * remove typo quote * create tools/generate_docker_image_names.py * pysen run format * update usage * fix B950 * Bashの行コメント"#"をdocstringに変えたときに消し忘れていたので削除 * コメントの表現修正: タグ名 -> Dockerイメージ名 * コメントの表現修正: タグ名 -> Dockerイメージ名 * コメントの表現修正: タグ名 -> Dockerイメージ名 * add --repository argument help * コメントの表現修正: 空文字 -> 空文字列 * helpの出力を末尾句点なしに統一 * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py * Update .github/workflows/build-engine-container.yml * Update tools/generate_docker_image_names.py * Update tools/generate_docker_image_names.py * Update tools/generate_docker_image_names.py * pysen run format * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py Co-authored-by: Hiroshiba * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * Update tools/generate_docker_image_names.py Co-authored-by: Hiroshiba * Update .github/workflows/build-engine-container.yml Co-authored-by: Hiroshiba * rename matrix.tags -> matrix.prefixes * rename buildcache_tag -> buildcache_prefix * lockファイルに更新があったっぽいので追従 * generate_docker_image_names: styling arguments * Update .github/workflows/build-engine-container.yml --------- Co-authored-by: Hiroshiba Co-authored-by: Hiroshiba Kazuyuki --- .github/workflows/build-engine-container.yml | 114 +++++++++--------- tools/generate_docker_image_names.py | 117 +++++++++++++++++++ 2 files changed, 172 insertions(+), 59 deletions(-) create mode 100644 tools/generate_docker_image_names.py diff --git a/.github/workflows/build-engine-container.yml b/.github/workflows/build-engine-container.yml index 60ee4d254..2151dd941 100644 --- a/.github/workflows/build-engine-container.yml +++ b/.github/workflows/build-engine-container.yml @@ -35,59 +35,44 @@ jobs: build-docker: needs: [config] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: - os: [ubuntu-latest] - tag: - - "" - - cpu - - cpu-ubuntu20.04 - - nvidia - - nvidia-ubuntu20.04 - - cpu-ubuntu22.04 - - nvidia-ubuntu22.04 + # 各変数の説明 + # prefixes: Docker tagのプレフィックス。カンマ区切り。空文字列の場合、バージョン文字列のみがタグ名になる + # buildcache_prefix: ビルドキャッシュのプレフィックス。空文字列やカンマは使用不可 + # target: Dockerfileのビルドステージ名 + # base_image: Dockerfileのビルド用ステージのベースイメージ + # base_runtime_image: Dockerfileの実行用ステージのベースイメージ + # onnxruntime_version: ONNX Runtimeのバージョン + # platforms: Dockerのプラットフォームバリアント。カンマ区切り。 参考: https://docs.docker.com/build/building/multi-platform/ include: # Ubuntu 20.04 - - tag: "" + - prefixes: ",cpu,cpu-ubuntu20.04" + buildcache_prefix: "cpu-ubuntu20.04" target: runtime-env base_image: ubuntu:20.04 base_runtime_image: ubuntu:20.04 onnxruntime_version: 1.13.1 platforms: linux/amd64,linux/arm64/v8 - - tag: cpu - target: runtime-env - base_image: ubuntu:20.04 - base_runtime_image: ubuntu:20.04 - onnxruntime_version: 1.13.1 - platforms: linux/amd64,linux/arm64/v8 - - tag: cpu-ubuntu20.04 - target: runtime-env - base_image: ubuntu:20.04 - base_runtime_image: ubuntu:20.04 - onnxruntime_version: 1.13.1 - platforms: linux/amd64,linux/arm64/v8 - - tag: nvidia - target: runtime-nvidia-env - base_image: ubuntu:20.04 - base_runtime_image: nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu20.04 - onnxruntime_version: 1.13.1 - platforms: linux/amd64 - - tag: nvidia-ubuntu20.04 + - prefixes: "nvidia,nvidia-ubuntu20.04" + buildcache_prefix: "nvidia-ubuntu20.04" target: runtime-nvidia-env base_image: ubuntu:20.04 base_runtime_image: nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu20.04 onnxruntime_version: 1.13.1 platforms: linux/amd64 # Ubuntu 22.04 - - tag: cpu-ubuntu22.04 + - prefixes: "cpu-ubuntu22.04" + buildcache_prefix: "cpu-ubuntu22.04" target: runtime-env base_image: ubuntu:22.04 base_runtime_image: ubuntu:22.04 onnxruntime_version: 1.13.1 platforms: linux/amd64,linux/arm64/v8 - - tag: nvidia-ubuntu22.04 + - prefixes: "nvidia-ubuntu22.04" + buildcache_prefix: "nvidia-ubuntu22.04" target: runtime-nvidia-env base_image: ubuntu:22.04 base_runtime_image: nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 @@ -137,32 +122,43 @@ jobs: DOWNLOAD_RESOURCE_PATH: download/resource run: bash tools/process_voicevox_resource.bash + - name: Generate Docker image names + id: generate-docker-image-names + run: | + # Dockerイメージ名を outputs.tags に改行区切りで格納する + { + echo "tags<> "$GITHUB_OUTPUT" + + # ビルドキャッシュに指定するためのDockerイメージ名を生成する + # NOTE: デフォルトブランチへのコミットの場合のみキャッシュを作成する + - name: Generate Docker buildcache image names + id: generate-docker-buildcache-image-names + run: | + # --cache-from に指定するためのDockerイメージ名 + # 常にデフォルトブランチのビルドキャッシュ（*-latest-buildcache）を使用する + cache_from="type=registry,ref=${{ env.IMAGE_NAME }}:${{ matrix.buildcache_prefix }}-latest-buildcache" + + # --cache-to に指定するためのDockerイメージ名 + # リリースの場合、ビルドキャッシュを作成しないため、空文字列を格納する + cache_to="" + if [ "${{ needs.config.outputs.version_or_latest }}" = "latest" ]; then + cache_to="type=registry,ref=${{ env.IMAGE_NAME }}:${{ matrix.buildcache_prefix }}-latest-buildcache,mode=max" + fi + + # outputs に格納する + echo "cache-from=$cache_from" >> "$GITHUB_OUTPUT" + echo "cache-to=$cache_to" >> "$GITHUB_OUTPUT" + - name: Build and Deploy Docker image uses: docker/build-push-action@v5 - env: - IMAGE_TAG: - |- # If it's a release, add the version, otherwise add the `latest` - ${{ ( - matrix.tag != '' && ( - format('{0}:{1}-{2}', env.IMAGE_NAME, matrix.tag, needs.config.outputs.version_or_latest) - ) || format('{0}:{1}', env.IMAGE_NAME, needs.config.outputs.version_or_latest) - ) }} - IMAGE_CACHE_FROM: - |- # Always use the `latest` buildcache. :latest-buildcache or :{tag}-latest-buildcache - ${{ ( - matrix.tag != '' && ( - format('type=registry,ref={0}:{1}-latest-buildcache', env.IMAGE_NAME, matrix.tag) - ) || format('type=registry,ref={0}:latest-buildcache', env.IMAGE_NAME) - ) }} - IMAGE_CACHE_TO: - |- # If it's a release, do not create buildcache, otherwise create the `latest` buildcache. :latest-buildcache or :{tag}-latest-buildcache - ${{ ( - needs.config.outputs.version_or_latest == 'latest' && ( - matrix.tag != '' && ( - format('type=registry,ref={0}:{1}-latest-buildcache,mode=max', env.IMAGE_NAME, matrix.tag) - ) || format('type=registry,ref={0}:latest-buildcache,mode=max', env.IMAGE_NAME) - ) || '' - ) }} with: context: . builder: ${{ steps.buildx.outputs.name }} @@ -178,9 +174,9 @@ jobs: ONNXRUNTIME_VERSION=${{ matrix.onnxruntime_version }} target: ${{ matrix.target }} push: true - tags: ${{ env.IMAGE_TAG }} - cache-from: ${{ env.IMAGE_CACHE_FROM }} - cache-to: ${{ env.IMAGE_CACHE_TO }} + tags: ${{ steps.generate-docker-image-names.outputs.tags }} + cache-from: ${{ steps.generate-docker-buildcache-image-names.outputs.cache-from }} + cache-to: ${{ steps.generate-docker-buildcache-image-names.outputs.cache-to }} platforms: ${{ matrix.platforms }} run-release-test-workflow: diff --git a/tools/generate_docker_image_names.py b/tools/generate_docker_image_names.py new file mode 100644 index 000000000..c3abc34b9 --- /dev/null +++ b/tools/generate_docker_image_names.py @@ -0,0 +1,117 @@ +""" +Dockerリポジトリ名、バージョン文字列、カンマ区切りのプレフィックスを受け取り、 +バージョン文字列付きのDockerイメージ名を改行区切りで標準出力に出力する + +例 +$ python3 ./tools/generate_docker_image_names.py \ + --repository "REPO" \ + --version "VER" \ + --prefix ",A,B" +REPO:VER +REPO:A-VER +REPO:B-VER + +$ python3 ./tools/generate_docker_image_names.py \ + --repository "REPO" \ + --version "VER" \ + --prefix "" +REPO:VER +""" + +from argparse import ArgumentParser + + +def generate_docker_image_names( + repository: str, + version: str, + comma_separated_prefix: str, +) -> list[str]: + """ + Dockerリポジトリ名、バージョン文字列、カンマ区切りのプレフィックスを受け取り、 + バージョン文字列付きのDockerイメージ名を配列で返す + + prefixが空文字列でない場合、"{prefix}-{version}"をタグにする + + - 例: repository="REPO", version="VER", prefix="A" -> "REPO:A-VER" + + prefixが空文字列の場合、"{version}"をタグにする + + - 例: repository="REPO", version="VER", prefix="" -> "REPO:VER" + + Parameters + ---------- + repository : str + Dockerリポジトリ名 + version : str + バージョン文字列 + comma_separated_prefix : str + カンマ区切りのプレフィックス + + Returns + ------- + list[str] + Dockerイメージ名の配列。 + + Examples + -------- + >>> generate_docker_image_names("voicevox/voicevox_engine", "0.22.0", "cpu,cpu-ubuntu22.04") + ['voicevox/voicevox_engine:0.22.0', + 'voicevox/voicevox_engine:cpu-0.22.0', + 'voicevox/voicevox_engine:cpu-ubuntu22.04-0.22.0'] + """ + # カンマ区切りのタグ名を配列に変換 + prefixes = comma_separated_prefix.split(",") + + # 戻り値の配列 + docker_image_names: list[str] = [] + + for prefix in prefixes: + # プレフィックスが空文字列でない場合、末尾にハイフンを付ける + if prefix: + prefix = f"{prefix}-" + docker_image_names.append(f"{repository}:{prefix}{version}") + + return docker_image_names + + +def main() -> None: + parser = ArgumentParser() + parser.add_argument( + "--repository", + type=str, + required=True, + help="Dockerリポジトリ名（例: voicevox/voicevox_engine）", + ) + parser.add_argument( + "--version", + type=str, + default="latest", + help='バージョン文字列（例: "0.22.0", "latest"）', + ) + parser.add_argument( + "--prefix", + type=str, + default="", + help='カンマ区切りのプレフィックス（例: ",cpu,cpu-ubuntu22.04", "nvidia,nvidia-ubuntu22.04"）', + ) + + args = parser.parse_args() + + repository: str = args.repository + version: str = args.version + comma_separated_prefix: str = args.prefix + + # Dockerイメージ名を生成 + docker_image_names = generate_docker_image_names( + repository=repository, + version=version, + comma_separated_prefix=comma_separated_prefix, + ) + + # 標準出力に出力 + for docker_image_name in docker_image_names: + print(docker_image_name) + + +if __name__ == "__main__": + main()