diff --git a/.github/workflows/docker_build_images.yml b/.github/workflows/build_docker_images.yml similarity index 63% rename from .github/workflows/docker_build_images.yml rename to .github/workflows/build_docker_images.yml index a923b7f57c3..0daada540b8 100644 --- a/.github/workflows/docker_build_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -1,22 +1,101 @@ -name: Docker Build Images +name: Build Docker Images on: + pull_request: push: branches: - main tags: - - 'v[2-9][0-9]*.*' # run only on tags greater or equal to v20.0.0 where this new way of building docker image was changed + - '*' concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build Images (v20+)') + group: format('{0}-{1}', ${{ github.ref }}, 'Build Docker Images') cancel-in-progress: true permissions: read-all jobs: + push: + name: Set push variable + runs-on: ubuntu-20.04 + if: github.repository == 'vitessio/vitess' + outputs: + push: ${{ steps.push.outputs.push }} + + steps: + - name: Set push variable + id: push + run: | + push='false' + if [[ "${{github.event.pull_request}}" == "" ]]; then + push='true' + fi + echo Push='${push}' + echo "push=${push}" >> $GITHUB_OUTPUT + + build_and_push_vttestserver: + name: Build and push vttestserver + runs-on: gh-hosted-runners-16cores-1 + if: github.repository == 'vitessio/vitess' && needs.push.result == 'success' + needs: + - push + + strategy: + fail-fast: true + matrix: + branch: [ mysql80 ] + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Login to Docker Hub + if: needs.push.outputs.push == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on main + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: ${{ needs.push.outputs.push }} + tags: vitess/vttestserver:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_lite: - name: Build and push vitess/lite Docker images + name: Build and push lite runs-on: gh-hosted-runners-16cores-1 - if: github.repository == 'vitessio/vitess' + if: github.repository == 'vitessio/vitess' && needs.push.result == 'success' + needs: + - push strategy: fail-fast: true @@ -28,6 +107,7 @@ jobs: uses: actions/checkout@v4 - name: Login to Docker Hub + if: needs.push.outputs.push == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -42,12 +122,12 @@ jobs: fi - name: Build and push on main - if: github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/tags/') == false uses: docker/build-push-action@v5 with: context: . file: ${{ env.DOCKERFILE }} - push: true + push: ${{ needs.push.outputs.push }} tags: vitess/lite:${{ matrix.branch }} ###### @@ -76,10 +156,12 @@ jobs: tags: ${{ env.DOCKER_TAG }} build_and_push_components: - name: Build and push vitess components Docker images - needs: build_and_push_lite + name: Build and push runs-on: gh-hosted-runners-16cores-1 - if: github.repository == 'vitessio/vitess' + if: github.repository == 'vitessio/vitess' && needs.push.result == 'success' && needs.build_and_push_lite.result == 'success' + needs: + - push + - build_and_push_lite strategy: fail-fast: true @@ -92,6 +174,7 @@ jobs: uses: actions/checkout@v4 - name: Login to Docker Hub + if: needs.push.outputs.push == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -102,22 +185,22 @@ jobs: echo "DOCKER_CTX=./docker/binaries/${{ matrix.component }}" >> $GITHUB_ENV - name: Build and push on main latest tag - if: github.ref == 'refs/heads/main' && matrix.debian == 'bookworm' + if: startsWith(github.ref, 'refs/tags/') == false && matrix.debian == 'bookworm' uses: docker/build-push-action@v5 with: context: ${{ env.DOCKER_CTX }} - push: true + push: ${{ needs.push.outputs.push }} tags: vitess/${{ matrix.component }}:latest build-args: | VT_BASE_VER=latest DEBIAN_VER=${{ matrix.debian }}-slim - name: Build and push on main debian specific tag - if: github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/tags/') == false uses: docker/build-push-action@v5 with: context: ${{ env.DOCKER_CTX }} - push: true + push: ${{ needs.push.outputs.push }} tags: vitess/${{ matrix.component }}:latest-${{ matrix.debian }} build-args: | VT_BASE_VER=latest diff --git a/.github/workflows/docker_build_vttestserver.yml b/.github/workflows/docker_build_vttestserver.yml deleted file mode 100644 index 1d157eef921..00000000000 --- a/.github/workflows/docker_build_vttestserver.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Docker Build vttestserver -on: - push: - branches: - - main - tags: - - '*' - -concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Docker Build vttestserver') - cancel-in-progress: true - -permissions: read-all - -jobs: - build_and_push: - name: Build and push vitess/vttestserver Docker images - runs-on: gh-hosted-runners-16cores-1 - if: github.repository == 'vitessio/vitess' - - strategy: - fail-fast: true - matrix: - branch: [ mysql80 ] - - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set Dockerfile path - run: | - echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV - - - name: Build and push on main - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ env.DOCKERFILE }} - push: true - tags: vitess/vttestserver:${{ matrix.branch }} - - - name: Get the Git tag - if: startsWith(github.ref, 'refs/tags/') - run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Set Docker tag name - if: startsWith(github.ref, 'refs/tags/') - run: | - echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV - - - name: Build and push on tags - if: startsWith(github.ref, 'refs/tags/') - uses: docker/build-push-action@v5 - with: - context: . - file: ${{ env.DOCKERFILE }} - push: true - tags: ${{ env.DOCKER_TAG }} diff --git a/Makefile b/Makefile index f0b9038a769..e46f5d1ca5a 100644 --- a/Makefile +++ b/Makefile @@ -334,7 +334,7 @@ docker_lite: docker_mini: ${call build_docker_image,docker/mini/Dockerfile,vitess/mini} -DOCKER_VTTESTSERVER_SUFFIX = mysql57 mysql80 +DOCKER_VTTESTSERVER_SUFFIX = mysql80 DOCKER_VTTESTSERVER_TARGETS = $(addprefix docker_vttestserver_,$(DOCKER_VTTESTSERVER_SUFFIX)) $(DOCKER_VTTESTSERVER_TARGETS): docker_vttestserver_%: ${call build_docker_image,docker/vttestserver/Dockerfile.$*,vitess/vttestserver:$*} diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index fd01f7d9ef6..7c0b22aed3c 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -12,29 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: We have to build the Vitess binaries from scratch instead of sharing -# a base image because Docker Hub dropped the feature we relied upon to -# ensure images contain the right binaries. - -# Use a temporary layer for the build stage. -ARG bootstrap_version=34 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" - -FROM "${image}" AS builder +FROM --platform=linux/amd64 golang:1.22.5-bullseye AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER -# Re-copy sources from working tree. -COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess +WORKDIR /vt/src/vitess.io/vitess -# Build and install Vitess in a temporary output directory. +# Create vitess user +RUN groupadd -r vitess && useradd -r -g vitess vitess +RUN mkdir -p /vt/vtdataroot /home/vitess +RUN chown -R vitess:vitess /vt /home/vitess USER vitess +# Re-copy sources from working tree. +COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess + RUN make install PREFIX=/vt/install # Start over and build the final image. -FROM debian:bullseye-slim +FROM --platform=linux/amd64 debian:bullseye-slim # Install dependencies COPY docker/utils/install_dependencies.sh /vt/dist/install_dependencies.sh @@ -45,7 +42,7 @@ RUN groupadd -r vitess && useradd -r -g vitess vitess RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt/src/vitess.io/vitess +ENV VTROOT /vt ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 1e27f45e692..cded55438ec 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -12,28 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: We have to build the Vitess binaries from scratch instead of sharing -# a base image because Docker Hub dropped the feature we relied upon to -# ensure images contain the right binaries. - -# Use a temporary layer for the build stage. -ARG bootstrap_version=34 -ARG image="vitess/bootstrap:${bootstrap_version}-percona80" - -FROM "${image}" AS builder +FROM --platform=linux/amd64 golang:1.22.5-bullseye AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER +WORKDIR /vt/src/vitess.io/vitess + +# Create vitess user +RUN groupadd -r vitess && useradd -r -g vitess vitess +RUN mkdir -p /vt/vtdataroot /home/vitess +RUN chown -R vitess:vitess /vt /home/vitess +USER vitess + # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess -# Build and install Vitess in a temporary output directory. -USER vitess RUN make install PREFIX=/vt/install # Start over and build the final image. -FROM debian:bullseye-slim +FROM --platform=linux/amd64 debian:bullseye-slim # Install dependencies COPY docker/utils/install_dependencies.sh /vt/dist/install_dependencies.sh @@ -44,7 +42,7 @@ RUN groupadd -r vitess && useradd -r -g vitess vitess RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt/src/vitess.io/vitess +ENV VTROOT /vt ENV VTDATAROOT /vt/vtdataroot ENV PATH $VTROOT/bin:$PATH diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile deleted file mode 100644 index 5e9d4aed326..00000000000 --- a/docker/local/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG bootstrap_version=34 -ARG image="vitess/bootstrap:${bootstrap_version}-common" - -FROM "${image}" - -RUN apt-get update -RUN apt-get install -y sudo curl vim jq - -# Install dependencies -COPY docker/utils/install_dependencies.sh /vt/dist/install_dependencies.sh -RUN /vt/dist/install_dependencies.sh mysql80 - -COPY docker/local/install_local_dependencies.sh /vt/dist/install_local_dependencies.sh -RUN /vt/dist/install_local_dependencies.sh -RUN echo "source /vt/common/env.sh" >> /etc/bash.bashrc - -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 - -# Re-copy sources from working tree. -COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess - -# Build and install Vitess in a temporary output directory. -USER vitess - -WORKDIR /vt/src/vitess.io/vitess -RUN make install PREFIX=/vt/install - -ENV VTROOT /vt/src/vitess.io/vitess -ENV VTDATAROOT /vt/vtdataroot -ENV PATH $VTROOT/bin:$PATH -ENV PATH="/var/opt/etcd:${PATH}" - -RUN mkdir /vt/local -COPY examples/local /vt/local - -# Copy the vtadmin web app to the correct location and npm install -COPY --chown=vitess:vitess web /web -RUN npm install /web/vtadmin -RUN /web/vtadmin/build.sh - -RUN mkdir /vt/common -COPY examples/common /vt/common - -CMD cd /vt/local && ./101_initial_cluster.sh && /bin/bash diff --git a/docker/local/install_local_dependencies.sh b/docker/local/install_local_dependencies.sh deleted file mode 100755 index 07fd302e283..00000000000 --- a/docker/local/install_local_dependencies.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# This is a script that gets run as part of the Dockerfile build -# to install dependencies for the vitess/mini image. -# -# Usage: install_local_dependencies.sh - -set -euo pipefail - -# Install etcd -ETCD_VER=v3.4.9 -DOWNLOAD_URL=https://storage.googleapis.com/etcd - -curl -k -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -mkdir -p /var/opt/etcd -sudo tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /var/opt/etcd --strip-components=1 -rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz - -mkdir -p /var/run/etcd && chown -R vitess:vitess /var/run/etcd - -# Clean up files we won't need in the final image. -rm -rf /var/lib/apt/lists/* - -# Install npm and node dependencies for vtadmin -curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && apt-get install -y nodejs diff --git a/docker/local/run.sh b/docker/local/run.sh deleted file mode 100755 index 16b07fc426c..00000000000 --- a/docker/local/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -docker run -d -p 14200:14200 -p 14201:14201 -p 15000:15000 -p 15001:15001 -p 15991:15991 -p 15999:15999 -p 16000:16000 --rm -it vitess/local diff --git a/docker/vttestserver/Dockerfile.mysql80 b/docker/vttestserver/Dockerfile.mysql80 index d6b0a200fa7..2e79d389c7d 100644 --- a/docker/vttestserver/Dockerfile.mysql80 +++ b/docker/vttestserver/Dockerfile.mysql80 @@ -12,28 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: We have to build the Vitess binaries from scratch instead of sharing -# a base image because Docker Hub dropped the feature we relied upon to -# ensure images contain the right binaries. - -# Use a temporary layer for the build stage. -ARG bootstrap_version=34 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" - -FROM "${image}" AS builder +FROM --platform=linux/amd64 golang:1.22.5-bullseye AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER +WORKDIR /vt/src/vitess.io/vitess + +# Create vitess user +RUN groupadd -r vitess && useradd -r -g vitess vitess +RUN mkdir -p /vt/vtdataroot /home/vitess +RUN chown -R vitess:vitess /vt /home/vitess +USER vitess + # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess -# Build and install Vitess in a temporary output directory. -USER vitess RUN make install-testing PREFIX=/vt/install # Start over and build the final image. -FROM debian:bullseye-slim +FROM --platform=linux/amd64 debian:bullseye-slim # Install dependencies COPY docker/utils/install_dependencies.sh /vt/dist/install_dependencies.sh diff --git a/go/tools/go-upgrade/go-upgrade.go b/go/tools/go-upgrade/go-upgrade.go index ee42040e5c0..34543120202 100644 --- a/go/tools/go-upgrade/go-upgrade.go +++ b/go/tools/go-upgrade/go-upgrade.go @@ -393,6 +393,9 @@ func replaceGoVersionInCodebase(old, new *version.Version, workflowUpdate bool) "./test/templates", "./build.env", "./docker/bootstrap/Dockerfile.common", + "./docker/lite/Dockerfile", + "./docker/lite/Dockerfile.percona80", + "./docker/vttestserver/Dockerfile.mysql80", } if workflowUpdate { explore = append(explore, "./.github/workflows") @@ -435,9 +438,6 @@ func updateBootstrapVersionInCodebase(old, new string, newGoVersion *version.Ver return nil } files, err := getListOfFilesInPaths([]string{ - "./docker/lite", - "./docker/local", - "./docker/vttestserver", "./Makefile", "./test/templates", })