From f313e742439a4abf452e1b86609b4079d0c525d4 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 21:28:49 +0900 Subject: [PATCH 01/11] fix bug --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 19b8415..10634fd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,7 +19,6 @@ env: JAX_CONDA_IMAGE_NAME : "yfukai/conda-jax" IMAGE_NAME: ${{ github.repository }} - jobs: build: @@ -63,7 +62,7 @@ jobs: id: meta_jax_conda uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }} - name: Extract Docker metadata id: meta From 3571feac26e44df6d0042d20f41c06d6a397f7c1 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 21:30:28 +0900 Subject: [PATCH 02/11] deleted duplicate --- .../workflows/jax-conda-docker-publish.yml | 128 ------------------ 1 file changed, 128 deletions(-) delete mode 100644 .github/workflows/jax-conda-docker-publish.yml diff --git a/.github/workflows/jax-conda-docker-publish.yml b/.github/workflows/jax-conda-docker-publish.yml deleted file mode 100644 index 66034d6..0000000 --- a/.github/workflows/jax-conda-docker-publish.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Docker - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -on: - release: - types: [ "published" ] - pull_request: - branches: [ "main" ] - -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - JAX_VERSION : 0.4.23 - # github.repository as / - JAX_CONDA_IMAGE_NAME : "yfukai/conda-jax" - IMAGE_NAME: ${{ github.repository }} - - -jobs: - build: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - # This is used to complete the identity challenge - # with sigstore/fulcio when running outside of PRs. - id-token: write - - strategy: - matrix: - target: ["cpu", "gpu"] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Install the cosign tool except on PR - # https://github.com/sigstore/cosign-installer - - name: Install Cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v3.1.1 - with: - cosign-release: 'v2.2.1' - - # Workaround: https://github.com/docker/build-push-action/issues/461 - - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Get the tag name - run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action - - name: Build and push Docker image (JAX conda CPU) - id: build-and-push-jax-conda-cpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64,linux/arm64/v8 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_version }} - - - name: Build and push Docker image (JAX conda GPU) - id: build-and-push-jax-conda-gpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_version }} - JAX_VERSION_EXTRA="cuda11_local" - BASE_IMAGE="nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04" - - - name: Build and push Docker image (CPU) - id: build-and-push-cpu - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64/v8 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - BASE_IMAGE="${{ env.REGISTRY }}/yfukai/conda-jax:latest" - - - name: Build and push Docker image (GPU) - id: build-and-push-gpu - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - BASE_IMAGE="${{ env.REGISTRY }}/yfukai/conda-jax:latest-cuda" From e0f9fa45a0674adc621bbec99cdca5a387b36ed2 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 21:36:20 +0900 Subject: [PATCH 03/11] fix env name --- .github/workflows/docker-publish.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 10634fd..ed8e11f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,9 +71,6 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Get the tag name - run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -89,7 +86,7 @@ jobs: tags: ${{ steps.meta_jax_conda.outputs.tags }} labels: ${{ steps.meta_jax_conda.outputs.labels }} build-args: | - JAX_VERSION=${{ env.JAX_version }} + JAX_VERSION=${{ env.JAX_VERSION }} - name: Build and push Docker image (JAX conda GPU) id: build-and-push-jax-conda-gpu @@ -101,7 +98,7 @@ jobs: tags: ${{ steps.meta_jax_conda.outputs.tags }}-cuda labels: ${{ steps.meta_jax_conda.outputs.labels }} build-args: | - JAX_VERSION=${{ env.JAX_version }} + JAX_VERSION=${{ env.JAX_VERSION }} JAX_VERSION_EXTRA="cuda11_local" BASE_IMAGE="nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04" From fc2d49b2d69d2605a86ec486e518edaab8b06318 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 22:08:51 +0900 Subject: [PATCH 04/11] fixed arg bug --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ed8e11f..bc5cf9b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -99,8 +99,8 @@ jobs: labels: ${{ steps.meta_jax_conda.outputs.labels }} build-args: | JAX_VERSION=${{ env.JAX_VERSION }} - JAX_VERSION_EXTRA="cuda11_local" - BASE_IMAGE="nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04" + JAX_VERSION_EXTRA=cuda11_local + BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 - name: Build and push Docker image (CPU) id: build-and-push-cpu @@ -112,7 +112,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE="${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest" + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest - name: Build and push Docker image (GPU) id: build-and-push-gpu @@ -124,4 +124,4 @@ jobs: tags: ${{ steps.meta.outputs.tags }}-cuda labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE="${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest-cuda" + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest-cuda From 7aecdac1dca35deb04f06744dda876fe583c13ea Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 22:09:14 +0900 Subject: [PATCH 05/11] test --- .github/workflows/docker-publish.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index bc5cf9b..4d07064 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -76,17 +76,17 @@ jobs: # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - - name: Build and push Docker image (JAX conda CPU) - id: build-and-push-jax-conda-cpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64,linux/arm64/v8 - push: true #${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta_jax_conda.outputs.tags }} - labels: ${{ steps.meta_jax_conda.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_VERSION }} +# - name: Build and push Docker image (JAX conda CPU) +# id: build-and-push-jax-conda-cpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64,linux/arm64/v8 +# push: true #${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta_jax_conda.outputs.tags }} +# labels: ${{ steps.meta_jax_conda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} - name: Build and push Docker image (JAX conda GPU) id: build-and-push-jax-conda-gpu From 554801506caf4d4ce59f45302f8d5aad980906d3 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Mon, 29 Jan 2024 22:14:47 +0900 Subject: [PATCH 06/11] test2 --- .github/workflows/docker-publish.yml | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4d07064..fbb635a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -88,20 +88,20 @@ jobs: # build-args: | # JAX_VERSION=${{ env.JAX_VERSION }} - - name: Build and push Docker image (JAX conda GPU) - id: build-and-push-jax-conda-gpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64 - push: true #${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta_jax_conda.outputs.tags }}-cuda - labels: ${{ steps.meta_jax_conda.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_VERSION }} - JAX_VERSION_EXTRA=cuda11_local - BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 - +# - name: Build and push Docker image (JAX conda GPU) +# id: build-and-push-jax-conda-gpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64 +# push: true #${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta_jax_conda.outputs.tags }}-cuda +# labels: ${{ steps.meta_jax_conda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} +# JAX_VERSION_EXTRA=cuda11_local +# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 +# - name: Build and push Docker image (CPU) id: build-and-push-cpu uses: docker/build-push-action@v5 @@ -112,7 +112,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:pr-8 - name: Build and push Docker image (GPU) id: build-and-push-gpu @@ -124,4 +124,4 @@ jobs: tags: ${{ steps.meta.outputs.tags }}-cuda labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:latest-cuda + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:pr-8-cuda From 05fd626263c0981bb3fe0b2440914b980222c422 Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Tue, 30 Jan 2024 09:38:30 +0900 Subject: [PATCH 07/11] finished except deleting image --- .../workflows/delete-docker-on-pr-close.yml | 155 ++++++++++++++++++ .github/workflows/docker-publish.yml | 123 +++++++++----- 2 files changed, 240 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/delete-docker-on-pr-close.yml diff --git a/.github/workflows/delete-docker-on-pr-close.yml b/.github/workflows/delete-docker-on-pr-close.yml new file mode 100644 index 0000000..2a6d526 --- /dev/null +++ b/.github/workflows/delete-docker-on-pr-close.yml @@ -0,0 +1,155 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + pull_request: + branches: [ "main" ] + types: [ closed ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + JAX_VERSION : 0.4.23 + JAX_CONDA_IMAGE_NAME : "yfukai/conda-jax" + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install Cosign +# if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@v3.1.1 + with: + cosign-release: 'v2.2.1' + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} +# if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata (JAX conda CPU) + id: meta_jax_conda + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + + - name: Extract Docker metadata (JAX conda GPU) + id: meta_jax_conda_cuda + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }} + tags: | + type=schedule,suffix=-cuda + type=ref,event=branch,suffix=-cuda + type=ref,event=tag,suffix=-cuda + type=ref,event=pr,suffix=-cuda + + - name: Extract Docker metadata (CPU) + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + + - name: Extract Docker metadata (GPU) + id: meta_cuda + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=schedule,suffix=-cuda + type=ref,event=branch,suffix=-cuda + type=ref,event=tag,suffix=-cuda + type=ref,event=pr,suffix=-cuda + +# # Build and push Docker image with Buildx +# # https://github.com/docker/build-push-action +# - name: Build and push Docker image (JAX conda CPU) +# id: build-and-push-jax-conda-cpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64,linux/arm64/v8 +# push: true #${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta_jax_conda.outputs.tags }} +# labels: ${{ steps.meta_jax_conda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} +# +# - name: Build and push Docker image (JAX conda GPU) +# id: build-and-push-jax-conda-gpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64 +# push: true #${{ github.event_name != 'pull_request' }} +# tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} +# labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} +# JAX_VERSION_EXTRA=cuda11_local +# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 +# +# - name: Build and push Docker image (CPU) +# id: build-and-push-cpu +# uses: docker/build-push-action@v5 +# with: +# context: . +# platforms: linux/amd64,linux/arm64/v8 +# push: true +# tags: ${{ steps.meta.outputs.tags }} +# labels: ${{ steps.meta.outputs.labels }} +# build-args: | +# BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_first_tag }} +# +# - name: Build and push Docker image (GPU) +# id: build-and-push-gpu +# uses: docker/build-push-action@v5 +# with: +# context: . +# platforms: linux/amd64 +# push: true +# tags: ${{ steps.meta_cuda.outputs.tags }} +# labels: ${{ steps.meta_cuda.outputs.labels }} +# build-args: | +# BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_cuda_first_tag }} +# diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fbb635a..727cd53 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,8 +15,8 @@ env: # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io JAX_VERSION : 0.4.23 - # github.repository as / JAX_CONDA_IMAGE_NAME : "yfukai/conda-jax" + # github.repository as / IMAGE_NAME: ${{ github.repository }} jobs: @@ -58,61 +58,108 @@ jobs: # Extract metadata (tags, labels) for Docker # https://github.com/docker/metadata-action - - name: Extract Docker metadata (JAX conda) + - name: Extract Docker metadata (JAX conda CPU) id: meta_jax_conda uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 with: images: ${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }} - - - name: Extract Docker metadata + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + + - name: Extract first tag (JAX conda CPU) + id: extract_first_tag_cpu + run: | + # Extracting the first component + first_component=$(echo "${{ steps.meta_jax_conda.outputs.tags }}" | cut -d ',' -f 1) + # Setting the output + echo "meta_jax_conda_first_tag=$first_component" >> $GITHUB_OUTPUT + + - name: Extract Docker metadata (JAX conda GPU) + id: meta_jax_conda_cuda + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }} + tags: | + type=schedule,suffix=-cuda + type=ref,event=branch,suffix=-cuda + type=ref,event=tag,suffix=-cuda + type=ref,event=pr,suffix=-cuda + + - name: Extract first tag (JAX conda GPU) + id: extract_first_tag_gpu + run: | + # Extracting the first component + first_component=$(echo "${{ steps.meta_jax_conda_cuda.outputs.tags }}" | cut -d ',' -f 1) + # Setting the output + echo "meta_jax_conda_cuda_first_tag=$first_component" >> $GITHUB_OUTPUT + + - name: Extract Docker metadata (CPU) id: meta uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + + - name: Extract Docker metadata (GPU) + id: meta_cuda + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=schedule,suffix=-cuda + type=ref,event=branch,suffix=-cuda + type=ref,event=tag,suffix=-cuda + type=ref,event=pr,suffix=-cuda - name: Set up QEMU uses: docker/setup-qemu-action@v2 - # Build and push Docker image with Buildx (don't push on PR) + # Build and push Docker image with Buildx # https://github.com/docker/build-push-action -# - name: Build and push Docker image (JAX conda CPU) -# id: build-and-push-jax-conda-cpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64,linux/arm64/v8 -# push: true #${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta_jax_conda.outputs.tags }} -# labels: ${{ steps.meta_jax_conda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} - -# - name: Build and push Docker image (JAX conda GPU) -# id: build-and-push-jax-conda-gpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64 -# push: true #${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta_jax_conda.outputs.tags }}-cuda -# labels: ${{ steps.meta_jax_conda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} -# JAX_VERSION_EXTRA=cuda11_local -# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 -# + - name: Build and push Docker image (JAX conda CPU) + id: build-and-push-jax-conda-cpu + uses: docker/build-push-action@v5 + with: + context: jax_image/ + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: ${{ steps.meta_jax_conda.outputs.tags }} + labels: ${{ steps.meta_jax_conda.outputs.labels }} + build-args: | + JAX_VERSION=${{ env.JAX_VERSION }} + + - name: Build and push Docker image (JAX conda GPU) + id: build-and-push-jax-conda-gpu + uses: docker/build-push-action@v5 + with: + context: jax_image/ + platforms: linux/amd64 + push: true + tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} + labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} + build-args: | + JAX_VERSION=${{ env.JAX_VERSION }} + JAX_VERSION_EXTRA=cuda11_local + BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 + - name: Build and push Docker image (CPU) id: build-and-push-cpu uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64/v8 - push: true # ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:pr-8 + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_first_tag }} - name: Build and push Docker image (GPU) id: build-and-push-gpu @@ -120,8 +167,8 @@ jobs: with: context: . platforms: linux/amd64 - push: true #${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }}-cuda - labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta_cuda.outputs.tags }} + labels: ${{ steps.meta_cuda.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:pr-8-cuda + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_cuda_first_tag }} From 64ee8887f32edf3366f8141178149ac516e162cd Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Tue, 30 Jan 2024 09:44:34 +0900 Subject: [PATCH 08/11] added delete docker on pr close --- .../workflows/delete-docker-on-pr-close.yml | 83 +++++++------------ 1 file changed, 31 insertions(+), 52 deletions(-) diff --git a/.github/workflows/delete-docker-on-pr-close.yml b/.github/workflows/delete-docker-on-pr-close.yml index 2a6d526..0d2ba3c 100644 --- a/.github/workflows/delete-docker-on-pr-close.yml +++ b/.github/workflows/delete-docker-on-pr-close.yml @@ -101,55 +101,34 @@ jobs: type=ref,event=tag,suffix=-cuda type=ref,event=pr,suffix=-cuda -# # Build and push Docker image with Buildx -# # https://github.com/docker/build-push-action -# - name: Build and push Docker image (JAX conda CPU) -# id: build-and-push-jax-conda-cpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64,linux/arm64/v8 -# push: true #${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta_jax_conda.outputs.tags }} -# labels: ${{ steps.meta_jax_conda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} -# -# - name: Build and push Docker image (JAX conda GPU) -# id: build-and-push-jax-conda-gpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64 -# push: true #${{ github.event_name != 'pull_request' }} -# tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} -# labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} -# JAX_VERSION_EXTRA=cuda11_local -# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 -# -# - name: Build and push Docker image (CPU) -# id: build-and-push-cpu -# uses: docker/build-push-action@v5 -# with: -# context: . -# platforms: linux/amd64,linux/arm64/v8 -# push: true -# tags: ${{ steps.meta.outputs.tags }} -# labels: ${{ steps.meta.outputs.labels }} -# build-args: | -# BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_first_tag }} -# -# - name: Build and push Docker image (GPU) -# id: build-and-push-gpu -# uses: docker/build-push-action@v5 -# with: -# context: . -# platforms: linux/amd64 -# push: true -# tags: ${{ steps.meta_cuda.outputs.tags }} -# labels: ${{ steps.meta_cuda.outputs.labels }} -# build-args: | -# BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_cuda_first_tag }} -# + - name: Delete image (JAX conda CPU) + uses: bots-house/ghcr-delete-image-action@v1.1.0 + with: + owner: yfukai + name: ${{ env.JAX_CONDA_IMAGE_NAME }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.meta_jax_conda.outputs.tags }} + + - name: Delete image (JAX conda GPU) + uses: bots-house/ghcr-delete-image-action@v1.1.0 + with: + owner: yfukai + name: ${{ env.JAX_CONDA_IMAGE_NAME }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.meta_jax_conda_cuda.outputs.tags }} + + - name: Delete image (JAX conda CPU) + uses: bots-house/ghcr-delete-image-action@v1.1.0 + with: + owner: yfukai + name: ${{ env.IMAGE_NAME }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.meta.outputs.tags }} + + - name: Delete image (JAX conda GPU) + uses: bots-house/ghcr-delete-image-action@v1.1.0 + with: + owner: yfukai + name: ${{ env.IMAGE_NAME }} + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.meta_cuda.outputs.tags }} From e608a793b2a6efcfe2f725c0fe341f824c1e53ac Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Tue, 30 Jan 2024 10:10:29 +0900 Subject: [PATCH 09/11] test --- .github/workflows/docker-publish.yml | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 727cd53..c31bdc0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -123,31 +123,31 @@ jobs: # Build and push Docker image with Buildx # https://github.com/docker/build-push-action - - name: Build and push Docker image (JAX conda CPU) - id: build-and-push-jax-conda-cpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64,linux/arm64/v8 - push: true - tags: ${{ steps.meta_jax_conda.outputs.tags }} - labels: ${{ steps.meta_jax_conda.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_VERSION }} - - - name: Build and push Docker image (JAX conda GPU) - id: build-and-push-jax-conda-gpu - uses: docker/build-push-action@v5 - with: - context: jax_image/ - platforms: linux/amd64 - push: true - tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} - labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} - build-args: | - JAX_VERSION=${{ env.JAX_VERSION }} - JAX_VERSION_EXTRA=cuda11_local - BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 +# - name: Build and push Docker image (JAX conda CPU) +# id: build-and-push-jax-conda-cpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64,linux/arm64/v8 +# push: true +# tags: ${{ steps.meta_jax_conda.outputs.tags }} +# labels: ${{ steps.meta_jax_conda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} +# +# - name: Build and push Docker image (JAX conda GPU) +# id: build-and-push-jax-conda-gpu +# uses: docker/build-push-action@v5 +# with: +# context: jax_image/ +# platforms: linux/amd64 +# push: true +# tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} +# labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} +# build-args: | +# JAX_VERSION=${{ env.JAX_VERSION }} +# JAX_VERSION_EXTRA=cuda11_local +# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 - name: Build and push Docker image (CPU) id: build-and-push-cpu @@ -159,7 +159,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_first_tag }} + BASE_IMAGE=${{ env.meta_jax_conda_first_tag }} - name: Build and push Docker image (GPU) id: build-and-push-gpu @@ -171,4 +171,4 @@ jobs: tags: ${{ steps.meta_cuda.outputs.tags }} labels: ${{ steps.meta_cuda.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/${{ env.JAX_CONDA_IMAGE_NAME }}:${{ env.meta_jax_conda_cuda_first_tag }} + BASE_IMAGE=${{ env.meta_jax_conda_cuda_first_tag }} From 20a563d4ec3b415d73ec430a1236e1ba26fc022b Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Tue, 30 Jan 2024 10:14:09 +0900 Subject: [PATCH 10/11] fixed bug --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c31bdc0..1255c11 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -159,7 +159,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.meta_jax_conda_first_tag }} + BASE_IMAGE=${{ steps.extract_first_tag_cpu.outputs.meta_jax_conda_first_tag }} - name: Build and push Docker image (GPU) id: build-and-push-gpu @@ -171,4 +171,4 @@ jobs: tags: ${{ steps.meta_cuda.outputs.tags }} labels: ${{ steps.meta_cuda.outputs.labels }} build-args: | - BASE_IMAGE=${{ env.meta_jax_conda_cuda_first_tag }} + BASE_IMAGE=${{ steps.extract_first_tag_gpu.outputs.meta_jax_conda_cuda_first_tag }} From b6e5d84e49f7c4267df56491dc37305c1d8bd32f Mon Sep 17 00:00:00 2001 From: Yohsuke Fukai Date: Tue, 30 Jan 2024 10:24:51 +0900 Subject: [PATCH 11/11] check all working --- .../workflows/delete-docker-on-pr-close.yml | 2 - .github/workflows/docker-publish.yml | 52 +++++++++---------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.github/workflows/delete-docker-on-pr-close.yml b/.github/workflows/delete-docker-on-pr-close.yml index 0d2ba3c..b474481 100644 --- a/.github/workflows/delete-docker-on-pr-close.yml +++ b/.github/workflows/delete-docker-on-pr-close.yml @@ -36,7 +36,6 @@ jobs: # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - name: Install Cosign -# if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@v3.1.1 with: cosign-release: 'v2.2.1' @@ -48,7 +47,6 @@ jobs: # Login against a Docker registry except on PR # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} -# if: github.event_name != 'pull_request' uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c with: registry: ${{ env.REGISTRY }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1255c11..bbaf403 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,7 +37,6 @@ jobs: # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - name: Install Cosign -# if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@v3.1.1 with: cosign-release: 'v2.2.1' @@ -49,7 +48,6 @@ jobs: # Login against a Docker registry except on PR # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} -# if: github.event_name != 'pull_request' uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c with: registry: ${{ env.REGISTRY }} @@ -123,31 +121,31 @@ jobs: # Build and push Docker image with Buildx # https://github.com/docker/build-push-action -# - name: Build and push Docker image (JAX conda CPU) -# id: build-and-push-jax-conda-cpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64,linux/arm64/v8 -# push: true -# tags: ${{ steps.meta_jax_conda.outputs.tags }} -# labels: ${{ steps.meta_jax_conda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} -# -# - name: Build and push Docker image (JAX conda GPU) -# id: build-and-push-jax-conda-gpu -# uses: docker/build-push-action@v5 -# with: -# context: jax_image/ -# platforms: linux/amd64 -# push: true -# tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} -# labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} -# build-args: | -# JAX_VERSION=${{ env.JAX_VERSION }} -# JAX_VERSION_EXTRA=cuda11_local -# BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 + - name: Build and push Docker image (JAX conda CPU) + id: build-and-push-jax-conda-cpu + uses: docker/build-push-action@v5 + with: + context: jax_image/ + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: ${{ steps.meta_jax_conda.outputs.tags }} + labels: ${{ steps.meta_jax_conda.outputs.labels }} + build-args: | + JAX_VERSION=${{ env.JAX_VERSION }} + + - name: Build and push Docker image (JAX conda GPU) + id: build-and-push-jax-conda-gpu + uses: docker/build-push-action@v5 + with: + context: jax_image/ + platforms: linux/amd64 + push: true + tags: ${{ steps.meta_jax_conda_cuda.outputs.tags }} + labels: ${{ steps.meta_jax_conda_cuda.outputs.labels }} + build-args: | + JAX_VERSION=${{ env.JAX_VERSION }} + JAX_VERSION_EXTRA=cuda11_local + BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 - name: Build and push Docker image (CPU) id: build-and-push-cpu