release-prepare #21
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: release-prepare | |
| on: | |
| workflow_dispatch: | |
| create: | |
| branches: [misc/prepare-release-*] | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| fetch-versions: | |
| if: github.repository == 'zenml-io/zenml' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| old_version: ${{ steps.old-version.outputs.old_version }} | |
| new_version: ${{ steps.new-version.outputs.new_version }} | |
| steps: | |
| # Extract the version | |
| - name: Extract version from branch name | |
| id: new-version | |
| run: | | |
| BRANCH_NAME=${GITHUB_REF#refs/heads/} | |
| NEW_VERSION=${BRANCH_NAME#misc/prepare-release-} | |
| echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT | |
| # Check out main to get the old version | |
| - name: Checkout code | |
| id: checkout-code | |
| uses: actions/[email protected] | |
| with: | |
| ref: main | |
| # Configure Git | |
| - name: Configure git | |
| shell: bash | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "ZenML GmbH" | |
| # Extract the old version | |
| - name: Fetch the old version | |
| id: old-version | |
| run: | | |
| LATEST_RELEASE=$(gh release view --json tagName,publishedAt -q '{tag: .tagName, date: .publishedAt}') | |
| OLD_VERSION=$(echo "$LATEST_RELEASE" | jq -r .tag) | |
| echo "old_version=$OLD_VERSION" >> $GITHUB_OUTPUT | |
| prepare-changes: | |
| needs: fetch-versions | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Check out the code | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| # Configure Git | |
| - name: Configure git | |
| shell: bash | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "ZenML GmbH" | |
| # Validate the new version | |
| - name: Validate new version | |
| shell: bash | |
| run: | | |
| scripts/validate-new-version.sh ${{ needs.fetch-versions.outputs.new_version }} | |
| # Set up Python | |
| - name: Set up Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.12' | |
| # Install ZenML | |
| - name: Install ZenML and dependencies | |
| shell: bash | |
| run: | | |
| scripts/install-zenml-dev.sh --system --integrations "no" | |
| uv pip list | |
| uv pip check || true | |
| # Alembic migration file | |
| - name: Run Alembic merge | |
| shell: bash | |
| run: | | |
| alembic merge -m "Release" heads --rev-id ${{ needs.fetch-versions.outputs.new_version }} | |
| scripts/format.sh | |
| git add src/zenml/zen_stores/migrations/versions | |
| # Update the README, pyproject.toml, version and helm files | |
| - name: Update main files | |
| run: | | |
| sed -i "s/${{ needs.fetch-versions.outputs.old_version }}/${{ needs.fetch-versions.outputs.new_version }}/g" README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md | |
| git add README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md | |
| # Update the Quickstart references | |
| - name: Replace the references in the quickstart example | |
| run: | | |
| find examples/quickstart -type f \( -name "*.txt" -o -name "*.yaml" -o -name "*.ipynb" \) -print0 | | |
| while IFS= read -r -d '' file; do | |
| if [[ "$file" == *.ipynb ]]; then | |
| # For .ipynb files, we need to parse JSON | |
| jq --arg OLD ${{ needs.fetch-versions.outputs.old_version }} --arg NEW ${{ needs.fetch-versions.outputs.new_version }} \ | |
| '(.cells[] | select(.cell_type == "code") | .source) |= map(gsub($OLD; $NEW))' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file" | |
| else | |
| # For .txt and .yaml files, we can use sed | |
| sed -i "s/${{ needs.fetch-versions.outputs.old_version }}/${{ needs.fetch-versions.outputs.new_version }}/g" "$file" | |
| fi | |
| done | |
| git add examples/quickstart | |
| # Generate and append release notes | |
| - name: Generate release notes | |
| run: | | |
| RELEASE_NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes -F tag_name=${{ needs.fetch-versions.outputs.new_version }} -F target_commitish=${{ github.sha }} -F previous_tag_name=${{ needs.fetch-versions.outputs.old_version }} | jq -r '.body') | |
| { | |
| head -n 1 RELEASE_NOTES.md | |
| echo "" | |
| echo "# ${{ needs.fetch-versions.outputs.new_version }}" | |
| echo "" | |
| echo "$RELEASE_NOTES" | |
| echo "" | |
| tail -n +2 RELEASE_NOTES.md | |
| } > RELEASE_NOTES.md.new && mv RELEASE_NOTES.md.new RELEASE_NOTES.md | |
| git add RELEASE_NOTES.md | |
| # Push the changes | |
| - name: Push the changes | |
| run: | | |
| git commit -m "Adding the new version to the necessary files." | |
| git push origin HEAD:${{ github.ref }} | |
| # Create a PR | |
| - name: Create a pull request | |
| run: | | |
| gh pr create --base "develop" --head "${{ github.ref }}" \ | |
| --title "Prepare release ${{ needs.fetch-versions.outputs.new_version }}" \ | |
| --body "This PR prepares the release of version ${{ needs.fetch-versions.outputs.new_version }}." | |
| # Send a message to Discord to alert everyone for the release | |
| - name: Send message to Discord | |
| run: | | |
| curl -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -d '{ | |
| "content": "${{ github.event.sender.login }} is preparing the release .\n\n@growth and @product, please do not merge anything to develop until the process is completed.", | |
| "thread_name": "Preparing release: ${{ needs.fetch-versions.outputs.new_version }}" | |
| }' \ | |
| ${{ secrets.DISCORD_WEBHOOK_RELEASE }} | |
| build-test-images: | |
| runs-on: ubuntu-latest | |
| needs: [fetch-versions, prepare-changes] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| # Check out the prepare-release branch | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| # Sign in to Google | |
| - uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| service_account_email: ${{ secrets.GCP_CLOUDBUILD_EMAIL }} | |
| service_account_key: ${{ secrets.GCP_CLOUDBUILD_KEY }} | |
| project_id: ${{ secrets.GCP_CLOUDBUILD_PROJECT }} | |
| # Submit the Cloudbuild job | |
| - name: Build docker images | |
| run: | | |
| gcloud builds submit \ | |
| --quiet \ | |
| --config=release-cloudbuild-preparation.yaml \ | |
| --substitutions=_ZENML_BRANCH=${{ github.ref }},_ZENML_NEW_VERSION=${{ needs.fetch-versions.outputs.new_version }} | |
| # Sign in to AWS | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: arn:aws:iam::715803424590:role/gh-action-role-zenml-quickstart-ecr | |
| aws-region: eu-central-1 | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| run: | | |
| aws ecr get-login-password --region eu-central-1 | docker login 715803424590.dkr.ecr.eu-central-1.amazonaws.com --username AWS --password-stdin | |
| # Publish the AWS image | |
| - name: Pull quickstart image from Dockerhub | |
| run: | | |
| docker pull zenmldocker/prepare-release:quickstart-aws-${{ needs.fetch-versions.outputs.new_version }} | |
| - name: Push quickstart image to ECR | |
| run: | | |
| docker tag zenmldocker/prepare-release:quickstart-aws-${{ needs.fetch-versions.outputs.new_version }} 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ needs.fetch-versions.outputs.new_version }} | |
| docker push 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ needs.fetch-versions.outputs.new_version }} | |
| setup-prep-release-tenant: | |
| needs: [fetch-versions, build-test-images] | |
| env: | |
| ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }} | |
| ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Check out the code | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| # Setting up Python | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.12' | |
| # Install requests | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install requests | |
| # Deactivate and redeploy the tenant | |
| - name: Run tenant management script | |
| env: | |
| CLOUD_STAGING_CLIENT_ID: ${{ secrets.CLOUD_STAGING_CLIENT_ID }} | |
| CLOUD_STAGING_CLIENT_SECRET: ${{ secrets.CLOUD_STAGING_CLIENT_SECRET }} | |
| RELEASE_TENANT_ID: ${{ secrets.RELEASE_TENANT_ID }} | |
| ZENML_NEW_VERSION: ${{ needs.fetch-versions.outputs.new_version }} | |
| run: python scripts/redeploy-release-prep-tenant.py | |
| run_quickstart_pipelines: | |
| needs: [fetch-versions, setup-prep-release-tenant] | |
| runs-on: ubuntu-latest | |
| env: | |
| ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }} | |
| ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - cloud: aws | |
| parent_image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws | |
| - cloud: azure | |
| parent_image: zenmldocker/prepare-release:quickstart-azure | |
| - cloud: gcp | |
| parent_image: zenmldocker/prepare-release:quickstart-gcp | |
| steps: | |
| # Check out the code | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| # Setting up Python | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.12' | |
| # ZenML Integrations | |
| - name: Install ZenML and the required integrations | |
| run: | | |
| scripts/install-zenml-dev.sh --system --integrations "no" | |
| # Run the Quickstart pipeline | |
| - name: Run on ${{ matrix.cloud }} | |
| run: |- | |
| cd examples/quickstart | |
| zenml stack set ${{ matrix.cloud }} | |
| sed -i "s|parent_image:.*|parent_image: \"${{ matrix.parent_image }}-${{ needs.fetch-versions.outputs.new_version }}\"|" "configs/training_${{ matrix.cloud }}.yaml" | |
| sed -i 's|zenml\[server\]==[^[:space:]]*|git+https://github.com/zenml-io/zenml.git@${{ github.ref }}#egg=zenml[server]|g' requirements_${{ matrix.cloud }}.txt | |
| pip install -r requirements_${{ matrix.cloud }}.txt | |
| zenml integration install ${{ matrix.cloud }} -y | |
| python run.py --model_type=t5-small |