Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: singular build #2238

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 27 additions & 111 deletions .github/workflows/dagster-cloud-deploy.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# https://github.com/dagster-io/dagster-cloud-hybrid-quickstart
# https://github.com/dagster-io/dagster-cloud-action
# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners

name: Dagster Cloud Hybrid Deployment

Expand Down Expand Up @@ -29,16 +28,15 @@ env:
inputs.code-location }}
DAGSTER_CLOUD_YAML_PATH:
src/teamster/code_locations/${{ inputs.code-location }}/dagster-cloud.yaml
IMAGE_TAG:
${{ inputs.code-location }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
IMAGE_TAG: ${{ inputs.code-location }}-${{ github.sha }}-${{ github.run_id }}

permissions:
id-token: write
contents: read
pull-requests: write

jobs:
prerun:
dagster-cloud-deploy:
runs-on: ubuntu-latest
outputs:
result: ${{ steps.prerun.outputs.result }}
Expand All @@ -50,32 +48,18 @@ jobs:
id: prerun
uses: dagster-io/dagster-cloud-action/actions/utils/prerun@main

build:
needs:
- prerun
runs-on: ubuntu-latest
if: needs.prerun.outputs.result != 'skip'
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
- linux/arm64
steps:
- name: Prepare
run: |
platform=${{ matrix.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> "${GITHUB_ENV}"

# https://github.com/actions/checkout
- uses: actions/checkout@v4
- name: Checkout
uses: actions/checkout@v4
if: steps.prerun.outputs.result != 'skip'
with:
ref: ${{ github.head_ref }}

# https://github.com/google-github-actions/auth
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2
if: steps.prerun.outputs.result != 'skip'
with:
project_id: ${{ vars.GCP_PROJECT_ID }}
workload_identity_provider:
Expand All @@ -85,119 +69,51 @@ jobs:
# https://github.com/google-github-actions/setup-gcloud
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2
if: steps.prerun.outputs.result != 'skip'

- name: gcloud auth configure-docker
if: steps.prerun.outputs.result != 'skip'
run: gcloud auth configure-docker ${{ vars.GCP_REGION }}-docker.pkg.dev

# https://github.com/docker/metadata-action
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
if: steps.prerun.outputs.result != 'skip'
with:
images: ${{ env.REGISTRY_IMAGE }}

# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
if: steps.prerun.outputs.result != 'skip'
with:
platforms: |
linux/amd64
linux/arm64

# https://github.com/docker/build-push-action
- name: Build and push by digest
- name: Build and upload Docker image
id: build
uses: docker/build-push-action@v6
if: steps.prerun.outputs.result != 'skip'
with:
context: .
cache-from: type=gha,scope=build-${{ env.PLATFORM_PAIR }}
cache-to: type=gha,mode=max,scope=build-${{ env.PLATFORM_PAIR }}
platforms: ${{ matrix.platform }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.REGISTRY_IMAGE }}:${{ env.IMAGE_TAG }}
cache-from: type=gha
cache-to: type=gha,mode=max
labels: ${{ steps.meta.outputs.labels }}
build-args: |
PYTHON_VERSION=${{ inputs.python-version }}
CODE_LOCATION=${{ inputs.code-location }}
outputs:
type=image,name=${{ env.REGISTRY_IMAGE
}},push-by-digest=true,name-canonical=true,push=true

- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"

# https://github.com/actions/upload-artifact
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-${{ env.PLATFORM_PAIR }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1

merge:
runs-on: ubuntu-latest
needs:
- prerun
- build
if: needs.prerun.outputs.result != 'skip' && github.actor != 'dependabot[bot]'
steps:
# https://github.com/actions/download-artifact
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-*
merge-multiple: true

# https://github.com/google-github-actions/auth
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v2
with:
project_id: ${{ vars.GCP_PROJECT_ID }}
workload_identity_provider:
projects/${{ vars.GCP_PROJECT_NUMBER
}}/locations/global/workloadIdentityPools/github/providers/teamster

# https://github.com/google-github-actions/setup-gcloud
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v2

- name: gcloud auth configure-docker
run: gcloud auth configure-docker ${{ vars.GCP_REGION }}-docker.pkg.dev

# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Create manifest list and push
working-directory: /tmp/digests
run: |
docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ env.IMAGE_TAG }} $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)

- name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ env.IMAGE_TAG }}

deploy:
runs-on: ubuntu-latest
needs:
- prerun
- build
- merge
if: github.actor != 'dependabot[bot]'
steps:
# https://github.com/actions/checkout
- name: Checkout
uses: actions/checkout@v4
if: needs.prerun.outputs.result != 'skip'
with:
ref: ${{ github.head_ref }}

# Validate dagster_cloud.yaml and the connection to dagster.cloud
# https://github.com/dagster-io/dagster-cloud-action/blob/main/actions/utils/dagster-cloud-cli/action.yml
- name: Validate configuration
id: ci-validate
uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@main
if: needs.prerun.outputs.result != 'skip'
if: steps.prerun.outputs.result != 'skip'
with:
command:
ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{
Expand All @@ -209,7 +125,7 @@ jobs:
- name: Initialize build session
id: ci-init
uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@main
if: needs.prerun.outputs.result != 'skip'
if: steps.prerun.outputs.result != 'skip'
with:
project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
Expand All @@ -221,7 +137,7 @@ jobs:
- name: Update build session with image tag
id: ci-set-build-output
uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@main
if: needs.prerun.outputs.result != 'skip'
if: steps.prerun.outputs.result != 'skip'
with:
command:
ci set-build-output --location-name=${{ inputs.code-location }} --image-tag=${{
Expand All @@ -232,7 +148,7 @@ jobs:
- name: Deploy to Dagster Cloud
id: ci-deploy
uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@main
if: needs.prerun.outputs.result != 'skip'
if: steps.prerun.outputs.result != 'skip'
with:
command: ci deploy

Expand All @@ -242,7 +158,7 @@ jobs:
- name: Update PR comment for branch deployments
id: ci-notify
uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@main
if: needs.prerun.outputs.result != 'skip' && always()
if: steps.prerun.outputs.result != 'skip' && always()
with:
command: ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}

Expand All @@ -251,6 +167,6 @@ jobs:
- name: Generate a summary
id: ci-summary
uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@main
if: needs.prerun.outputs.result != 'skip' && always()
if: steps.prerun.outputs.result != 'skip' && always()
with:
command: ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY
Loading