From 972e8a5ab177d3bb86e2371eed1d3567d06e192f Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 5 Nov 2024 17:19:14 -0500 Subject: [PATCH] more CI changes --- .github/workflows/cannon.yml | 13 +++ .github/workflows/ci.yml | 31 +++++- .github/workflows/release.yml | 196 ++++++++++++++++++++++++---------- 3 files changed, 178 insertions(+), 62 deletions(-) diff --git a/.github/workflows/cannon.yml b/.github/workflows/cannon.yml index f6bf58c6bc541..ae7106fb99dbb 100644 --- a/.github/workflows/cannon.yml +++ b/.github/workflows/cannon.yml @@ -17,6 +17,19 @@ jobs: steps: - uses: actions/checkout@v4 + # Install just command + - name: Install just + run: | + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + + # Install Node.js (often needed for contract tooling) + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "18" + cache: "npm" + - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fda643bce306..a2baf7cb1ccdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,14 +46,35 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare Docker Tags + id: prep + run: | + # Get short SHA + SHA=$(git rev-parse --short HEAD) + + # Sanitize branch name by replacing invalid characters + BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed -e 's/[^a-zA-Z0-9._-]/-/g') + + # For pull requests, use PR number + if [[ $GITHUB_REF == refs/pull/* ]]; then + PR_NUMBER=$(echo $GITHUB_REF | cut -d / -f 3) + TAGS="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:pr-${PR_NUMBER}" + else + TAGS="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${SHA}" + # Add branch tag if not a PR + if [ "$BRANCH" != "" ]; then + TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${BRANCH}" + fi + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . push: ${{ github.event_name != 'pull_request' }} - tags: | - ${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${{ github.sha }} - ${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${{ github.ref_name }} + tags: ${{ steps.prep.outputs.tags }} platforms: linux/amd64 cache-from: type=gha cache-to: type=gha,mode=max @@ -61,7 +82,9 @@ jobs: - name: Save Docker image for testing if: github.event_name == 'pull_request' run: | - docker save ${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${{ github.sha }} > /tmp/${{ matrix.component }}.tar + # Extract first tag from the list + FIRST_TAG=$(echo "${{ steps.prep.outputs.tags }}" | cut -d',' -f1) + docker save "${FIRST_TAG}" > /tmp/${{ matrix.component }}.tar - name: Upload Docker image if: github.event_name == 'pull_request' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1450734db8c92..2df1c414f2e95 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ name: Release on: + # Manual trigger with required approval + workflow_dispatch: + # Automatic trigger on version tags push: tags: - "da-server/v*" @@ -15,84 +18,143 @@ env: REPO: ${{ github.repository_owner }} jobs: - docker-release: + # Wait for approval on manual release + approve-release: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + environment: release-approval + steps: + - run: echo "Release approved" + + docker-release-standard: + needs: [approve-release] + if: always() && (needs.approve-release.result == 'success' || needs.approve-release.result == 'skipped') runs-on: ubuntu-latest permissions: contents: read packages: write strategy: matrix: - include: + component: # Standard multi-platform components - - component: op-node - platforms: linux/amd64,linux/arm64 - - component: op-batcher - platforms: linux/amd64,linux/arm64 - - component: op-proposer - platforms: linux/amd64,linux/arm64 - - component: op-challenger - platforms: linux/amd64,linux/arm64 - - component: op-dispute-mon - platforms: linux/amd64,linux/arm64 - - component: op-conductor - platforms: linux/amd64,linux/arm64 - - component: da-server - platforms: linux/amd64,linux/arm64 - - component: op-ufm - platforms: linux/amd64,linux/arm64 - - component: op-supervisor - platforms: linux/amd64,linux/arm64 - - component: op-deployer - platforms: linux/amd64,linux/arm64 - - component: cannon - platforms: linux/amd64,linux/arm64 - # AMD64-only components that need larger runners - - component: ci-builder - platforms: linux/amd64 - runs-on: ubuntu-latest-xlarge - - component: ci-builder-rust - platforms: linux/amd64 - runs-on: ubuntu-latest-xlarge - - component: proofs-tools - platforms: linux/amd64 - runs-on: ubuntu-latest-xlarge - runs-on: ${{ matrix.runs-on || 'ubuntu-latest' }} + - op-node + - op-batcher + - op-proposer + - op-challenger + - op-dispute-mon + - op-conductor + - da-server + - op-ufm + - op-supervisor + - op-deployer + - cannon + steps: + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/*/v} + else + VERSION="latest" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }} + tags: | + type=sha + type=raw,value=${{ steps.version.outputs.version }} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + docker-release-amd64: + needs: [approve-release] + if: always() && (needs.approve-release.result == 'success' || needs.approve-release.result == 'skipped') + runs-on: ubuntu-latest-xlarge + permissions: + contents: read + packages: write + strategy: + matrix: + component: + # AMD64-only components + - ci-builder + - ci-builder-rust + - proofs-tools steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Extract version from tag + id: version + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/*/v} + else + VERSION="latest" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }} tags: | type=sha - type=ref,event=branch - type=ref,event=tag + type=raw,value=${{ steps.version.outputs.version }} + type=raw,value=latest,enable={{is_default_branch}} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true - platforms: ${{ matrix.platforms }} + platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - check-cross-platform: - needs: [docker-release] + verify-release: + needs: [docker-release-standard] runs-on: ubuntu-latest strategy: matrix: @@ -111,23 +173,41 @@ jobs: cannon, ] steps: - - name: Verify ARM64 Support - if: ${{ !contains(fromJson('["ci-builder", "ci-builder-rust", "proofs-tools"]'), matrix.component) }} + - name: Verify Multi-platform Support run: | - image_name="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${{ github.sha }}" + image_name="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:sha-${{ github.sha }}" - echo "Retrieving Docker image manifest: $image_name" - MANIFEST=$(docker manifest inspect $image_name) + echo "Verifying image: $image_name" - echo "Verifying 'linux/arm64' is supported..." - SUPPORTED_PLATFORM=$(echo "$MANIFEST" | jq -r '.manifests[] | select(.platform.architecture == "arm64" and .platform.os == "linux")') - if [ -z "$SUPPORTED_PLATFORM" ]; then - echo "Platform 'linux/arm64' not supported by this image" - exit 1 - fi + # Check for ARM64 support + echo "Checking ARM64 support..." + docker manifest inspect $image_name | jq -e '.manifests[] | select(.platform.architecture == "arm64")' + + # Check for AMD64 support + echo "Checking AMD64 support..." + docker manifest inspect $image_name | jq -e '.manifests[] | select(.platform.architecture == "amd64")' - name: Test Image run: | - image_name="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:${{ github.sha }}" + image_name="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.component }}:sha-${{ github.sha }}" docker pull $image_name - docker run $image_name ${{ matrix.component }} --version + docker run --rm $image_name ${{ matrix.component }} --version + + notify-release: + needs: [verify-release] + runs-on: ubuntu-latest + if: always() + steps: + - name: Check workflow status + id: check + run: | + if [ "${{ needs.verify-release.result }}" = "success" ]; then + echo "status=✅ Release successful" >> $GITHUB_OUTPUT + else + echo "status=❌ Release failed" >> $GITHUB_OUTPUT + fi + + - name: Create Release Summary + run: | + echo "### Release Status: ${{ steps.check.outputs.status }}" >> $GITHUB_STEP_SUMMARY + echo "* Release Tag: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY