From 1f393a49fc94be360b6bffe8be635e2ad1f06e3e Mon Sep 17 00:00:00 2001 From: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:46:09 -0600 Subject: [PATCH] [release-16.0] Build and push Docker Images from GitHub Actions (#14513) Signed-off-by: Florent Poinsard --- .github/workflows/docker_build_base.yml | 261 +++++++++++++++ .github/workflows/docker_build_lite.yml | 73 +++++ .../workflows/docker_build_vttestserver.yml | 65 ++++ doc/internal/ReleaseInstructions.md | 302 ++++++++---------- docker/release.sh | 73 ----- tools/back_to_dev_mode.sh | 1 - tools/create_release.sh | 1 - tools/release_utils.sh | 5 - 8 files changed, 529 insertions(+), 252 deletions(-) create mode 100644 .github/workflows/docker_build_base.yml create mode 100644 .github/workflows/docker_build_lite.yml create mode 100644 .github/workflows/docker_build_vttestserver.yml delete mode 100755 docker/release.sh diff --git a/.github/workflows/docker_build_base.yml b/.github/workflows/docker_build_base.yml new file mode 100644 index 00000000000..00848e2518e --- /dev/null +++ b/.github/workflows/docker_build_base.yml @@ -0,0 +1,261 @@ +name: Docker Build Base +on: + push: + branches: + - main + tags: + - '*' + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build Base') + cancel-in-progress: true + +permissions: read-all + +jobs: + build_and_push_base: + name: Build and push vitess/base Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ latest, mysql57, percona57, percona80 ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKERFILE=./docker/base/Dockerfile" >> $GITHUB_ENV + else + echo "DOCKERFILE=./docker/base/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on main + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/base:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') && matrix.branch == 'latest' + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKER_TAG=vitess/base:${TAG_NAME}" >> $GITHUB_ENV + fi + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') && matrix.branch == 'latest' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_k8s: + needs: build_and_push_base + name: Build and push vitess/k8s image + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + debian: [ bullseye, bookworm ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker context path + run: | + echo "DOCKER_CTX=./docker/k8s" >> $GITHUB_ENV + + - name: Build and push on main latest tag + if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/k8s:latest + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + - name: Build and push on main debian specific tag + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/k8s:latest-${{ matrix.debian }} + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + # We push git-tag-based k8s image to three tags, i.e. for 'v19.0.0' we push to: + # + # vitess/k8s:v19.0.0 (DOCKER_TAG_DEFAULT_DEBIAN) + # vitess/k8s:v19.0.0-bookworm (DOCKER_TAG) + # vitess/k8s:v19.0.0-bullseye (DOCKER_TAG) + # + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG_DEFAULT_DEBIAN=vitess/k8s:${TAG_NAME}" >> $GITHUB_ENV + echo "DOCKER_TAG=vitess/k8s:${TAG_NAME}-${{ matrix.debian }}" >> $GITHUB_ENV + + # Build and Push component image to DOCKER_TAG, applies to both debian version + - name: Build and push on tags using Debian extension + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + # Build and Push component image to DOCKER_TAG_DEFAULT_DEBIAN, only applies when building the default Debian version (bookworm) + # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already + - name: Build and push on tags without Debian extension + if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + + build_and_push_components: + needs: build_and_push_k8s + name: Build and push vitess components Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + debian: [ bullseye, bookworm ] + component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, logrotate, logtail, vtbackup, vtexplain ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker context path + run: | + echo "DOCKER_CTX=./docker/k8s/${{ matrix.component }}" >> $GITHUB_ENV + + - name: Build and push on main latest tag + if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + - name: Build and push on main debian specific tag + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest-${{ matrix.debian }} + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + # We push git-tag-based images to three tags, i.e. for 'v19.0.0' we push to: + # + # vitess/${{ matrix.component }}:v19.0.0 (DOCKER_TAG_DEFAULT_DEBIAN) + # vitess/${{ matrix.component }}:v19.0.0-bookworm (DOCKER_TAG) + # vitess/${{ matrix.component }}:v19.0.0-bullseye (DOCKER_TAG) + # + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG_DEFAULT_DEBIAN=vitess/${{ matrix.component }}:${TAG_NAME}" >> $GITHUB_ENV + echo "DOCKER_TAG=vitess/${{ matrix.component }}:${TAG_NAME}-${{ matrix.debian }}" >> $GITHUB_ENV + + # Build and Push component image to DOCKER_TAG, applies to both debian version + - name: Build and push on tags using Debian extension + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + # Build and Push component image to DOCKER_TAG_DEFAULT_DEBIAN, only applies when building the default Debian version (bookworm) + # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already + - name: Build and push on tags without Debian extension + if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' + uses: docker/build-push-action@v5 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim diff --git a/.github/workflows/docker_build_lite.yml b/.github/workflows/docker_build_lite.yml new file mode 100644 index 00000000000..7f355ddfd32 --- /dev/null +++ b/.github/workflows/docker_build_lite.yml @@ -0,0 +1,73 @@ +name: Docker Build Lite +on: + push: + branches: + - main + tags: + - '*' + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build Lite') + cancel-in-progress: true + +permissions: read-all + +jobs: + build_and_push: + name: Build and push vitess/lite Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ latest, mysql57, mysql80, percona57, percona80 ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKERFILE=./docker/lite/Dockerfile" >> $GITHUB_ENV + else + echo "DOCKERFILE=./docker/lite/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on main + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/lite:${{ matrix.branch }} + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKER_TAG=vitess/lite:${TAG_NAME}" >> $GITHUB_ENV + else + echo "DOCKER_TAG=vitess/lite:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} \ No newline at end of file diff --git a/.github/workflows/docker_build_vttestserver.yml b/.github/workflows/docker_build_vttestserver.yml new file mode 100644 index 00000000000..1223700527b --- /dev/null +++ b/.github/workflows/docker_build_vttestserver.yml @@ -0,0 +1,65 @@ +name: Docker Build vttestserver +on: + push: + branches: + - main + tags: + - '*' + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build vttestserver') + cancel-in-progress: true + +permissions: read-all + +jobs: + build_and_push: + name: Build and push vitess/vttestserver Docker images + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ mysql57, mysql80 ] + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on main + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/vttestserver:${{ matrix.branch }} + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} \ No newline at end of file diff --git a/doc/internal/ReleaseInstructions.md b/doc/internal/ReleaseInstructions.md index f45b9fb5902..fd5caa81e03 100644 --- a/doc/internal/ReleaseInstructions.md +++ b/doc/internal/ReleaseInstructions.md @@ -1,155 +1,53 @@ -# Release Instructions +# Release Cutover -This page describes the steps for cutting a new [open source release](https://github.com/vitessio/vitess/releases). +In this section we describe our current release process. Below is a summary of this document. -### Summary +- [**Pre-requisite for the release team**](#pre-requisites) +- [**Overview**](#overview) +- [**Pre-Release**](#pre-release) +- [**Release**](#release) +- [**Post-Release**](#post-release) +- [**How To prepare the release of Vitess**](#how-to-prepare-the-release-of-vitess) +- [**How To Release Vitess**](#how-to-release-vitess) +- [**How To Code Freeze**](#how-to-code-freeze) +- [**How To Merge During Code Freeze**](#how-to-merge-during-code-freeze) +- [**Java Packages: Deploy & Release**](#java-packages-deploy--release) -- [Versioning](#versioning) -- [Release Branches](#release-branches) -- [Release Tags](#release-tags) -- [Docker Images](#docker-images) -- [Java Packages](#java-packages) -- [Release Cutover](#release-cutover) -------- +----- -## Versioning - -Our versioning strategy is based on [VEP5](https://github.com/vitessio/enhancements/blob/main/veps/vep-5.md). - -### Major Release (vX) - -A new major release is needed when the public API changes in a -backward-incompatible way -- for example, when removing deprecated interfaces. - -Our public API includes (but is not limited to): - -* The VTGate [RPC interfaces](https://github.com/vitessio/vitess/tree/main/proto). -* The interfaces exposed by the VTGate client library in each language. - -Care must also be taken when changing the format of any data stored by a live -system, such as topology data or Vitess-internal tables (used for sequences, -distributed transactions, etc.). Although this data is considered as internal to -Vitess, if any change breaks the upgrade path for a live system (for example, -requiring that it be shut down and reinitialized from scratch), then it must be -considered as a breaking change. - -### Minor Release (vX.Y) - -A new minor release indicates that functionality has been added or changed in a -backward-compatible way. This should be the majority of normal releases. - -### Patch Release (vX.Y.Z) - -A patch release indicates that only a select set of bugfixes have been -cherry-picked onto the associated minor release. The expectation is that -upgrading by a patch release should be painless (not requiring any config -changes) and safe (isolated from active development on `main`). - -### Pre-Release Labels (vX.Y.Z-labelN) - -Pre-release versions should be labeled with a suffix like `-beta2` or `-rc1`. - -------- - -## Release Branches - -Each major and minor releases (X.Y) should have a [release branch](https://github.com/vitessio/vitess/branches/all?query=release) named -`release-X.Y`. This branch should diverge from `main` when the release -is declared, after which point only bugfix PRs should be cherry-picked onto the branch. -All other activity on `main` will go out with a subsequent major or minor release. - -```shell -git checkout main -git pull --ff-only upstream main - -git checkout -b release-X.Y -git push upstream release-X.Y -``` - -The branches are named `release-X.Y` to distinguish them from point-in-time -tags, which are named `vX.Y.Z`. - -------- - -## Release Tags - -While the release branch is a moving target, release tags mark point-in-time -snapshots of the repository. Essentially, a tag assigns a human-readable name to -a specific Git commit hash. Although it's technically possible to reassign a tag -name to a different hash, we must never do this. - -------- - -## Docker Images - -Docker images built automatically on DockerHub and can be found [here](https://hub.docker.com/repository/docker/vitess/lite/). - -------- - -## Java Packages - -We publish binary packages for our [JDBC driver and Java client on Maven Central](https://search.maven.org/#search|ga|1|g:"io.vitess"). - -To do so, we use the http://oss.sonatype.org/ repository. -New packages must be uploaded there ("deployed") and will be automatically published ("released"). -Once they are released there, they will be automatically synchronized with Maven Central. -The synchronization takes only several minutes, but the update on http://search.maven.org may take up to two hours. - -### Access to oss.sonatype.org - -[Sign up here.](https://issues.sonatype.org/secure/Signup!default.jspa) -Then you must be added as member to our `io.vitess` namespace. -Therefore, file a JIRA ticket with Sonatype to get added ([example for a different namespace](https://issues.sonatype.org/browse/OSSRH-30604)). - -### One-time setup - -#### Set up GPG - -Follow [Sonatype's GPG instructions](https://central.sonatype.org/pages/working-with-pgp-signatures.html). - -Install `gpg-agent` (needed below) e.g. on Ubuntu via: `sudo apt-get install gnupg-agent`. -for Mac you need to install 'gnupg' via 'brew install gnupg' - -#### Login configuration - -Create the `settings.xml` in the `$HOME/.m2/` directory as described in their [instructions](https://central.sonatype.org/pages/apache-maven.html). - -------- - -## Release Cutover - -In this section we describe our current release process. We begin with a list of [**pre-requisite for the release team**](#pre-requisites) and with a short [**overview**](#overview). -The release process is divided into three parts: [**Pre-Release**](#pre-release), [**Release**](#release), [**Post-Release**](#post-release), which are detailed after the overview. - -### Pre-Requisites +## Pre-Requisites This section highlights the different pre-requisites the release team has to meet before releasing. - The tool `gh` must be installed locally and ready to be used. -- You must have access to the Java release, more information in the [**Java Packages**](#java-packages) section. +- You must have access to the Java release, more information in the [**Java Packages**](./java-packages.md) section. - You must be able to create branches and have admin right on the `vitessio/vitess` and `planetscale/vitess-operator` repositories. -### Overview +----- + +## Overview -#### Schedule +### Schedule A new major version of Vitess is released every four months. For each major version there is at least one release candidate, which we release three weeks before the GA version. We usually create the RC1 during the first week of the month, and the GA version three weeks later. -#### Code Freeze +### Code Freeze Before creating RC1, there is a code freeze. Assuming the release of RC1 happens on a Tuesday, the release branch will be frozen Friday of the previous week. This allows us to test that the release branch can be released and avoid discovering unwanted events during the release day. Once the RC1 is released, there are three more weeks to backport bug fixes into the release branches. However, we also proceed to a code freeze the Friday before the GA release. (Assuming GA is on a Tuesday) Regarding patch releases, no code freeze is planned. -#### Tracking Issue for each Release +### Tracking Issue for each Release For each release, it is recommended to create an issue like [this one](https://github.com/vitessio/vitess/issues/10476) to track the current and past progress of a release. It also allows us to document what happened during a release. -### Pre-Release +----- + +## Pre-Release This step happens a few weeks before the actual release (whether it is an RC, GA or a patch release). The main goal of this step is to make sure everything is ready to be released for the release day. @@ -158,8 +56,9 @@ That includes: > - All the Pull Requests that need to be in the release must be reviewed and merged before the code freeze. > - The code freeze usually happens a few days before the release. - **Making sure the people doing the release have access to all the tools and infrastructure needed to do the release.** - > - This includes write access to the Vitess repository and to the Maven repository. + > - This includes write access to the Vitess repository and to the Maven repository. - **Preparing and cleaning the release notes summary.** + > - If the release does not contain significant changes (i.e. a small patch release) then this step can be skipped > - One or more Pull Requests have to be submitted in advance to create and update the release summary. > - The summary files are located in: `./changelog/*.0/*.*.*/summary.md`. > - The summary file for a release candidate is the same as the one for the GA release. @@ -171,6 +70,7 @@ That includes: > - As soon as we go into code freeze, if we are doing an RC, create the release branch. > - If we are doing a GA release, do not merge any new Pull Requests. > - The guide on how to do a code freeze is available in the [How To Code Freeze](#how-to-code-freeze) section. + > - It is not advised to merge a PR during code freeze, but if it is deemed necessary by the release lead, then follow the steps in [How To Merge During Code Freeze](#how-to-merge-during-code-freeze) section. - **Create the Vitess release.** > - A guide on how to create a Vitess release is available in the [How to prepare the release of Vitess](#how-to-prepare-the-release-of-vitess) section. > - This step will create a Release Pull Request, it must be reviewed and merged before the release day. The release commit will be used to tag the release. @@ -178,13 +78,33 @@ That includes: > - While the Vitess Operator is located in a different repository, we also need to do a release for it. > - The Operator follows the same cycle: RC1 -> GA -> Patches. > - Documentation for the pre-release of the Vitess Operator is available [here](https://github.com/planetscale/vitess-operator/blob/main/docs/release-process.md#prepare-for-release). - -### Release +- **Update the website documentation.** + > - We want to open a preparatory **draft** Pull Request to update the documentation. + > - There are several pages we want to update: + > - [The releases page](https://vitess.io/docs/releases/): we must add the new release to the list with all its information and link. The links can be broken (404 error) while we are preparing for the release, this is fine. + > - [The local install page](https://vitess.io/docs/get-started/local/): we must use the proper version increment for this guide and the proper SHA. The SHA will have to be modified once the Release Pull Request and the release is tagged is merged. + > - If we are doing a GA or RC release follow the instructions below: + > - There are two scripts in the website repository in `./tools/{ga|rc}_release.sh`, use them to update the website documentation. The scripts automate: + > - For an RC, we need to create a new entry in the sidebar which represents the next version on `main` and mark the version we are releasing as RC. + > - For a GA, we need to mark the version we are releasing as "Stable" and the next one as "Development". +- **Create a new GitHub Milestone** + > - Our GitHub Milestones is a good representation of all our ongoing development cycles. We have a Milestone for `main` and for all release branches. + > - After doing Code Freeze, we can create a new GitHub Milestone that matches the next development cycle. + > - **If we release a major version (v18.0.0-rc1):** we must create a `v19.0.0` Milestone. + > - **If we release a patch release (v17.0.3):** we must create a `v17.0.4` Milestone. + +----- + +## Release On the release day, there are several things to do: +- **Merge the Release Pull Request.** + > - During the code freeze, we created a Release Pull Request. It must be merged. - **Tag the Vitess release.** > - A guide on how to tag a version is available in the [How To Release Vitess](#how-to-release-vitess) section. +- **Update the release notes on `main`.** + > - One Pull Request against `main` must be created, it will contain the new release notes that we are adding in the Release Pull Request. - **Create the corresponding Vitess operator release.** > - Applies only to versions greater or equal to `v14.0.0`. > - If we are doing an RC release, then we will need to create the Vitess Operator RC too. If we are doing a GA release, we're also doing a GA release in the Operator. @@ -193,10 +113,9 @@ On the release day, there are several things to do: > - Applies only to GA releases. > - This step is explained in the [Java Packages: Deploy & Release](#java-packages-deploy--release) section. - **Update the website documentation repository.** - > - Applies only to GA and RC releases. - > - There are two scripts in the website repository in `./tools/{ga|rc}_release.sh`, use them to update the website documentation. The scripts automate: - > - For an RC, we need to create a new version in the sidebar and mark the current version as RC. - > - For a GA, we need to mark the version we are releasing as "Stable" and the next one as "Development". + > - Review the Website Pull Request that was opened during the Pre-Release. + > - The git SHA used in [the local install page](https://vitess.io/docs/get-started/local/) should be updated with the new proper SHA for this release. + > - Merge the Pull Request. - **Publish the blog post on the Vitess website.** > - Applies only to GA releases. > - The corresponding Pull Request was created beforehand during the pre-release. Merge it. @@ -206,21 +125,28 @@ On the release day, there are several things to do: > - After a while, those elements will finish their execution and their status will be green. > - This step is even more important for GA releases as we often include a link to _arewefastyet_ in the blog post. > - The benchmarks need to complete before announcing the blog posts or before they get cross-posted. -- **Update the release notes on `main`.** - > - One Pull Request against `main` must be created, it will contain the new release notes. - **Go back to dev mode on the release branch.** - > - The version constants across the codebase must be updated to `SNAPSHOT`. -- **Build k8s Docker images and publish them** - > - The docker image for `base`, `lite`, etc are built automatically by DockerHub. The k8s images however are dependent on these images and are required to be built manually. - > - These images should be built after the `base` image has been built and available on DockerHub. - > - To build and publish these images, run `./release.sh` from the directory `vitess/docker`. + > - The version constants across the codebase must be updated to `SNAPSHOT`. +- **Ensure the k8s images are available on DockerHub.** +- **Close the current GitHub Milestone** + > - Once we are done releasing the current version, we must close its corresponding GitHub Milestone as the development cycle for it is over. + > - **This does not apply if we are releasing an RC release.** For instance, if we are releasing `v18.0.0-rc1` we want to keep the `v18.0.0` milestone opened as development is not fully done for `v18.0.0`. + > - For instance, if we release `v18.0.1`, we must close the `v18.0.1` Milestone as the development cycle for `v18.0.1` is over. + > - When closing a Milestone, we need to look through all the PRs/Issues in that Milestone and re-assign a newer Milestone to them. + +----- -### Post-Release +## Post-Release Once the release is over, we need to announce it on both Slack and Twitter. We also want to make sure the blog post was cross-posted, if applicable. We need to verify that _arewefastyet_ has finished the benchmark too. -### How to prepare the release of Vitess +Moreover, once the roadmap discussions are over for the next release, we need to update the roadmap presented in the Vitess website (https://vitess.io/docs/resources/roadmap/). +We must remove everything that is now done in this release and add new items based on the discussions. + +----- + +## How to prepare the release of Vitess > In this example our current version is `v14.0.3` and we release the version `v15.0.0`. > Alongside Vitess' release, we also release a new version of the operator. @@ -239,8 +165,8 @@ We need to verify that _arewefastyet_ has finished the benchmark too. ``` 2. Creation of the Release Pull Request. - > This step will create the Release Pull Request that will then be reviewed ahead of the release day. - > The merge commit of that Pull Request will be used during the release day to tag the release. + > This step will create the Release Pull Request that will then be reviewed ahead of the release day. + > The merge commit of that Pull Request will be used during the release day to tag the release. 1. Run the `create_release` script using the Makefile: 1. Release Candidate: ```shell @@ -253,18 +179,15 @@ We need to verify that _arewefastyet_ has finished the benchmark too. The script will prompt you `Pausing so release notes can be added. Press enter to continue`. We are now going to generate the release notes, continue to the next sub-step. - 2. Run the following command to generate the release notes: - 1. Release Candidate: - ```shell - go run ./go/tools/release-notes --from "v14.0.3" --to "HEAD" --version "v15.0.0-rc1" --summary "./changelog/15.0/15.0.0/summary.md" [--threads=[0-9.]] - ``` - 2. General Availability: - ```shell - go run ./go/tools/release-notes --from "v14.0.3" --to "HEAD" --version "v15.0.0" --summary "./changelog/15.0/15.0.0/summary.md" [--threads=[0-9.]] - ``` - - > Important note: The release note generation fetches a lot of data from the GitHub API. You might reach the API request limit. - In which case you should use the `--threads=` flag and set an integer value lower than 10 (the default). + 2. Run the following command to generate the release notes. Note that you can omit the `--summary` flag if there are no summary. + ```shell + go run ./go/tools/release-notes --version "v15.0.0" --summary "./changelog/15.0/15.0.0/summary.md" + ``` + + > Make sure to also run `go run ./go/tools/releases/releases.go` to update the `./changelog` directory. + + > Important note: The release note generation fetches a lot of data from the GitHub API. You might reach the API request limit. + In which case you should use the `--threads=` flag and set an integer value lower than 10 (the default). This command will generate the release notes by looking at all the commits between the tag `v14.0.3` and the reference `HEAD`. It will also use the file located in `./changelog/15.0/15.0.0/summary.md` to prefix the release notes with a text that the maintainers wrote before the release. @@ -273,25 +196,29 @@ We need to verify that _arewefastyet_ has finished the benchmark too. 3. Follow the instruction prompted by the `create_release` Makefile command's output in order to push the newly created branch and create the Release Pull Request on GitHub. -### How To Release Vitess +4. If we are doing an RC release it means we created a new branch from `main`. We need to update `main` with the next SNAPSHOT version. If `main` was on `15.0.0-SNAPSHOT`, we need to update it to `16.0.0-SNAPSHOT`. A simple find and replace in the IDE is sufficient, there only a handful of files that must be changed: `version.go` and several java files. + +----- + +## How To Release Vitess This section is divided into two parts: - [Creation of the tags and release notes](#creation-of-the-tags-and-release-notes). - [Creating Release or Release Candidate on the GitHub UI](#creating-release-or-release-candidate-on-the-github-ui) -#### Creation of the tags and release notes +### Creation of the tags and release notes > This step implies that you have created a [Release Pull Request](#how-to-prepare-the-release-of-vitess) beforehand and that it has been reviewed. > The merge commit of this Release Pull Request will be used to tag the release. -> +> > In this example our current version is `v14.0.3` and we release the version `v15.0.0`. > Alongside Vitess' release, we also release a new version of the operator. > Since we are releasing a release candidate here, the new version of the operator will also be a release candidate. > In this example, the new operator version is `2.8.0`. -> +> > It is important to note that before the RC, there is a code freeze during which we create the release branch. > > The release branch in this example is `release-15.0`. -> +> > The example also assumes that `origin` is the `vitessio/vitess` remote. 1. Fetch `github.com/vitessio/vitess`'s remote. @@ -313,26 +240,26 @@ This section is divided into two parts: make BASE_BRANCH="release-15.0" BASE_REMOTE="origin" RELEASE_VERSION="15.0.0-rc1" DEV_VERSION="15.0.0-SNAPSHOT" back_to_dev_mode ``` > You will then need to follow the instructions given by the output of the back_to_dev_mode Makefile command. You will need to push the newly created branch and open a Pull Request. - + 6. Release the tag on GitHub UI as explained in the following section. -#### Creating Release or Release Candidate on the GitHub UI +### Creating Release or Release Candidate on the GitHub UI > In the below steps, we use `v8.0.0` and `v9.0.0` as an example. -##### 1. Open the releases page +#### 1. Open the releases page On Vitess' GitHub repository main page, click on Code -> [Releases](https://github.com/vitessio/vitess/releases). ![alt text](.images/release-01.png) -##### 2. Draft a new release +#### 2. Draft a new release On the Releases page, click on `Draft a new release`. ![alt text](.images/release-02.png) -##### 3. Tag a new release +#### 3. Tag a new release When drafting a new release, we are asked to choose the release's tag and branch. We format the tag this way: `v9.0.0`. We append `-rcN` to the tag name for release candidates, @@ -340,7 +267,7 @@ with `N` being the increment of the release candidate. ![alt text](.images/release-03.png) -##### 4. Add release notes and release +#### 4. Add release notes and release Copy/paste the previously built Release Notes into the description of the release. @@ -350,7 +277,9 @@ And finally, click on `Publish release`. ![alt text](.images/release-04.png) -### How To Code Freeze +----- + +## How To Code Freeze In this example we are going to do a code freeze on the `release-15.0` branch. If we are doing a release candidate, there won't be a branch yet, hence we need to create it. @@ -359,6 +288,9 @@ git fetch --all git checkout -b release-15.0 origin/main ``` +> Important: after creating the new branch `release-15.0`, we need to create new branch protection rules on the GitHub UI. +> The rules can be copied from the rules that are on the `main` branch. + The new branch will be based on `origin/main`, here `origin` points to `vitessio/vitess`. If we are not doing a release candidate, then the branch already exists and we can checkout on it. Now, if we are doing a GA release, let's update the branch: @@ -375,11 +307,35 @@ Finally, let's run the code freeze script: The script will prompt the command that will allow you to push the code freeze change. Once pushed, open a PR that will be merged on `release-15.0`. -### Java Packages: Deploy & Release +Remember, you should also disable the Launchable integration from the newly created release branch. + +----- + +## How To Merge During Code Freeze + +> **Warning:** It is not advised to merge a PR during code-freeze. If it is deemed absolutely necessary, then the following steps can be followed. + +The PR that needs to be merged will be failing on the `Code Freeze` CI. To merge this PR, we'll have to mark this CI action as not required. +You will need administrator privileges on the vitess repository to be able to make this change. + +1. Go to the GitHub repository and click on `Settings`. +2. Under the `Code and automation` section, select `Branches`. +3. Find the branch that you want to merge the PR against and then select `Edit`. +4. Scroll down to find the list of required checks. +5. Within this list find `Code Freeze` and click on the cross next to it to remove it from this list. +6. Save your changes on the bottom of the page. +7. Refresh the page of the PR, and you should be able to merge it. +8. After merging the PR, you need to do 2 more things - + 1. Add `Code Freeze` back as a required check. + 2. Check if the release PR has any merge conflicts. If it does, fix them and push. + +----- + +## Java Packages: Deploy & Release > **Warning:** This section's steps need to be executed only when releasing a new major version of Vitess, > or if the Java packages changed from one minor/patch version to another. -> +> > For this example, we assume we juste released `v12.0.0`. 1. Checkout to the release commit. @@ -387,7 +343,7 @@ The script will prompt the command that will allow you to push the code freeze c git checkout v12.0.0 ``` -2. Run `gpg-agent` to avoid that Maven will constantly prompt you for the password of your private key. +2. Run `gpg-agent` to avoid that Maven will constantly prompt you for the password of your private key. Note that this can print error messages that can be ignored on Mac. ```bash eval $(gpg-agent --daemon --no-grab --write-env-file $HOME/.gpg-agent-info) @@ -403,10 +359,12 @@ The script will prompt the command that will allow you to push the code freeze c 4. Deploy (upload) the Java code to the oss.sonatype.org repository: - > **Warning:** After the deployment, the Java packages will be automatically released. Once released, you cannot delete them. The only option is to upload a newer version (e.g. increment the patch level).

+ > **Warning:** After the deployment, the Java packages will be automatically released. Once released, you cannot delete them. The only option is to upload a newer version (e.g. increment the patch level).

```bash + cd ./java/ mvn clean deploy -P release -DskipTests cd .. ``` + 5. It will take some time for artifacts to appear on [maven directory](https://mvnrepository.com/artifact/io.vitess/vitess-client) diff --git a/docker/release.sh b/docker/release.sh deleted file mode 100755 index 7193dffe560..00000000000 --- a/docker/release.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -set -ex - -vt_base_version='v16.0.6' -debian_versions='buster bullseye' -default_debian_version='bullseye' - -docker pull --platform linux/amd64 vitess/base:$vt_base_version - -for debian_version in $debian_versions -do - echo "####### Building vitess/vt:$debian_version" - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/k8s:$vt_base_version-$debian_version k8s - docker tag vitess/k8s:$vt_base_version-$debian_version vitess/k8s:$vt_base_version - docker push vitess/k8s:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/k8s:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtadmin:$vt_base_version-$debian_version k8s/vtadmin - docker tag vitess/vtadmin:$vt_base_version-$debian_version vitess/vtadmin:$vt_base_version - docker push vitess/vtadmin:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtadmin:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtorc:$vt_base_version-$debian_version k8s/vtorc - docker tag vitess/vtorc:$vt_base_version-$debian_version vitess/vtorc:$vt_base_version - docker push vitess/vtorc:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtorc:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtgate:$vt_base_version-$debian_version k8s/vtgate - docker tag vitess/vtgate:$vt_base_version-$debian_version vitess/vtgate:$vt_base_version - docker push vitess/vtgate:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtgate:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vttablet:$vt_base_version-$debian_version k8s/vttablet - docker tag vitess/vttablet:$vt_base_version-$debian_version vitess/vttablet:$vt_base_version - docker push vitess/vttablet:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vttablet:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/mysqlctld:$vt_base_version-$debian_version k8s/mysqlctld - docker tag vitess/mysqlctld:$vt_base_version-$debian_version vitess/mysqlctld:$vt_base_version - docker push vitess/mysqlctld:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/mysqlctld:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/mysqlctl:$vt_base_version-$debian_version k8s/mysqlctl - docker tag vitess/mysqlctl:$vt_base_version-$debian_version vitess/mysqlctl:$vt_base_version - docker push vitess/mysqlctl:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/mysqlctl:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctl:$vt_base_version-$debian_version k8s/vtctl - docker tag vitess/vtctl:$vt_base_version-$debian_version vitess/vtctl:$vt_base_version - docker push vitess/vtctl:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctl:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctlclient:$vt_base_version-$debian_version k8s/vtctlclient - docker tag vitess/vtctlclient:$vt_base_version-$debian_version vitess/vtctlclient:$vt_base_version - docker push vitess/vtctlclient:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctlclient:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtctld:$vt_base_version-$debian_version k8s/vtctld - docker tag vitess/vtctld:$vt_base_version-$debian_version vitess/vtctld:$vt_base_version - docker push vitess/vtctld:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctld:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/logrotate:$vt_base_version-$debian_version k8s/logrotate - docker tag vitess/logrotate:$vt_base_version-$debian_version vitess/logrotate:$vt_base_version - docker push vitess/logrotate:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/logrotate:$vt_base_version; fi - - docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/logtail:$vt_base_version-$debian_version k8s/logtail - docker tag vitess/logtail:$vt_base_version-$debian_version vitess/logtail:$vt_base_version - docker push vitess/logtail:$vt_base_version-$debian_version - if [[ $debian_version == $default_debian_version ]]; then docker push vitess/logtail:$vt_base_version; fi -done diff --git a/tools/back_to_dev_mode.sh b/tools/back_to_dev_mode.sh index 7bbc6bce25f..27ad0234eff 100755 --- a/tools/back_to_dev_mode.sh +++ b/tools/back_to_dev_mode.sh @@ -48,7 +48,6 @@ fi function doBackToDevMode () { # Preparing the "dev mode" commit updateJava $DEV_VERSION - updateDockerReleaseScript $DEV_VERSION updateVersionGo $DEV_VERSION git add --all diff --git a/tools/create_release.sh b/tools/create_release.sh index 1c89696a427..62bdbe83051 100755 --- a/tools/create_release.sh +++ b/tools/create_release.sh @@ -61,7 +61,6 @@ function createRelease () { # Preparing the release commit updateVitessExamples $RELEASE_VERSION $VTOP_VERSION updateJava $RELEASE_VERSION - updateDockerReleaseScript $RELEASE_VERSION updateVersionGo $RELEASE_VERSION ## Create the commit for this release and tag it diff --git a/tools/release_utils.sh b/tools/release_utils.sh index ff7d348bcd3..d94d2fe7f31 100755 --- a/tools/release_utils.sh +++ b/tools/release_utils.sh @@ -24,11 +24,6 @@ function checkGitState() { fi } -function updateDockerReleaseScript () { - sed -i.bak -E "s/vt_base_version=.*/vt_base_version='v$1'/g" $ROOT/docker/release.sh - rm -f $ROOT/docker/release.sh.bak -} - function checkoutNewBranch () { branch_name=$1