From a853967dd07e2752c44f4001ffe889370a141be9 Mon Sep 17 00:00:00 2001 From: PARIYA ASHOK Date: Fri, 8 Nov 2024 16:45:59 +0530 Subject: [PATCH] Enable multi-platform support for operator image builds These changes enable building and pushing container images for multiple platforms (amd64, s390x, arm64) from a single Dockerfile. Enhanced multi-platform support in the build process by adding a PLATFORMS argument in the Makefile for amd64, s390x, and arm64 architectures. Updated the docker-build-operator target to support builds with docker buildx and podman, and added new targets for creating Docker Buildx builders and pushing multi-platform images. Signed-off-by: Ashok Pariya --- .dockerignore | 3 -- Makefile | 63 ++++++++++++++++++++++++++++++-- build/operator/Dockerfile | 30 +++++++++++++-- hack/build-multiarch-operator.sh | 61 +++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 10 deletions(-) create mode 100755 hack/build-multiarch-operator.sh diff --git a/.dockerignore b/.dockerignore index f32b68f25..45bc2bfd3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,9 +12,6 @@ _kubevirtci # No need for source code is already compiled -pkg -cmd -tools vendor docs diff --git a/Makefile b/Makefile index ce6092d57..373c1ddf4 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,19 @@ OPERATOR_IMAGE ?= cluster-network-addons-operator REGISTRY_IMAGE ?= cluster-network-addons-registry export OCI_BIN ?= $(shell if podman ps >/dev/null 2>&1; then echo podman; elif docker ps >/dev/null 2>&1; then echo docker; fi) TLS_SETTING := $(if $(filter $(OCI_BIN),podman),--tls-verify=false,) +PLATFORMS ?= linux/amd64 +# Set the platforms for building a multi-platform supported image. +# Example: +# PLATFORMS ?= linux/amd64,linux/arm64,linux/s390x +# Alternatively, you can export the PLATFORMS variable like this: +# export PLATFORMS=linux/arm64,linux/s390x,linux/amd64 +ARCH := $(shell uname -m | sed 's/x86_64/amd64/') +DOCKER_BUILDER ?= cnao-docker-builder +OPERATOR_IMAGE_TAGGED := $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) +NULL := +SPACE := $(NULL) # +COMMA := , +PLATFORM_LIST := $(subst $(COMMA),$(SPACE),$(strip $(PLATFORMS))) TARGETS = \ gen-k8s \ @@ -26,7 +39,6 @@ TARGETS = \ export GOFLAGS=-mod=vendor export GO111MODULE=on GO_VERSION = $(shell hack/go-version.sh) - WHAT ?= ./pkg/... ./cmd/... ./tools/... export E2E_TEST_TIMEOUT ?= 3h @@ -112,8 +124,14 @@ manifest-templator: $(GO) docker-build: docker-build-operator docker-build-registry -docker-build-operator: manager manifest-templator - $(OCI_BIN) build -f build/operator/Dockerfile -t $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) . +docker-build-operator: +ifeq ($(OCI_BIN),podman) + $(MAKE) build-multiarch-operator-podman +else ifeq ($(OCI_BIN),docker) + $(MAKE) build-multiarch-operator-docker +else + $(error Unsupported OCI_BIN value: $(OCI_BIN)) +endif docker-build-registry: $(OCI_BIN) build -f build/registry/Dockerfile -t $(IMAGE_REGISTRY)/$(REGISTRY_IMAGE):$(IMAGE_TAG) . @@ -121,7 +139,9 @@ docker-build-registry: docker-push: docker-push-operator docker-push-registry docker-push-operator: - $(OCI_BIN) push ${TLS_SETTING} $(IMAGE_REGISTRY)/$(OPERATOR_IMAGE):$(IMAGE_TAG) +ifeq ($(OCI_BIN),podman) + podman manifest push ${TLS_SETTING} ${OPERATOR_IMAGE_TAGGED} ${OPERATOR_IMAGE_TAGGED} +endif docker-push-registry: $(OCI_BIN) push $(IMAGE_REGISTRY)/$(REGISTRY_IMAGE):$(IMAGE_TAG) @@ -227,9 +247,44 @@ lint-monitoring: clean: rm -rf $(OUTPUT_DIR) +build-multiarch-operator-docker: +ifeq ($(words $(PLATFORM_LIST)), 1) + docker build \ + --build-arg BUILD_ARCH=$(ARCH) \ + --platform $(PLATFORMS) \ + -f build/operator/Dockerfile \ + -t ${OPERATOR_IMAGE_TAGGED} --push . +else + ./hack/build-multiarch-operator.sh ${DOCKER_BUILDER} + docker buildx build \ + --build-arg BUILD_ARCH=$(ARCH) \ + --platform $(PLATFORMS) \ + -f build/operator/Dockerfile \ + -t ${OPERATOR_IMAGE_TAGGED} --push . +endif + +build-multiarch-operator-podman: + # Remove any existing manifest and image + @podman manifest rm ${OPERATOR_IMAGE_TAGGED} || true + @podman rmi ${OPERATOR_IMAGE_TAGGED} || true + # Create a manifest list + @podman manifest create ${OPERATOR_IMAGE_TAGGED} + # Loop through each platform and build the image + $(foreach platform,$(PLATFORM_LIST), \ + podman build \ + --no-cache \ + --build-arg BUILD_ARCH=$(ARCH) \ + --platform $(platform) \ + --manifest ${OPERATOR_IMAGE_TAGGED} \ + -f build/operator/Dockerfile \ + .; \ + ) + .PHONY: \ $(E2E_SUITES) \ all \ + build-multiarch-operator-docker \ + build-multiarch-operator-podman \ check \ cluster-clean \ cluster-down \ diff --git a/build/operator/Dockerfile b/build/operator/Dockerfile index a134939ac..8557d9717 100644 --- a/build/operator/Dockerfile +++ b/build/operator/Dockerfile @@ -1,4 +1,25 @@ -FROM quay.io/centos/centos:stream9 +ARG BUILD_ARCH=amd64 +FROM --platform=linux/${BUILD_ARCH} quay.io/centos/centos:stream9 AS builder + +RUN dnf install -y tar gzip jq && dnf clean all +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/') && \ + GO_VERSION=$(curl -L -s "https://go.dev/dl/?mode=json" | jq -r '.[0].version') && \ + curl -L "https://go.dev/dl/${GO_VERSION}.linux-${ARCH}.tar.gz" -o go.tar.gz && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz +ENV PATH=$PATH:/usr/local/go/bin +WORKDIR /go/src/cluster-network-addons-operator +COPY . . + +ARG TARGETOS +ARG TARGETARCH +ARG TARGETPLATFORM + +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o build/_output/bin/manager ./cmd/... && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o build/_output/bin/manifest-templator ./tools/manifest-templator/... + +FROM --platform=linux/${TARGETARCH} quay.io/centos/centos:stream9 AS final + ENV ENTRYPOINT=/entrypoint \ OPERATOR=/cluster-network-addons-operator \ MANIFEST_TEMPLATOR=/manifest-templator \ @@ -8,13 +29,16 @@ ENV ENTRYPOINT=/entrypoint \ RUN \ yum -y update && \ yum clean all + COPY build/operator/bin/user_setup /user_setup COPY build/operator/bin/csv-generator /usr/bin/csv-generator COPY templates/cluster-network-addons/VERSION/cluster-network-addons-operator.VERSION.clusterserviceversion.yaml.in cluster-network-addons-operator.VERSION.clusterserviceversion.yaml.in RUN /user_setup COPY data /data -COPY build/_output/bin/manager $OPERATOR -COPY build/_output/bin/manifest-templator $MANIFEST_TEMPLATOR + +COPY --from=builder /go/src/cluster-network-addons-operator/build/_output/bin/manager $OPERATOR +COPY --from=builder /go/src/cluster-network-addons-operator/build/_output/bin/manifest-templator $MANIFEST_TEMPLATOR COPY build/operator/bin/entrypoint $ENTRYPOINT + ENTRYPOINT $ENTRYPOINT USER $USER_UID diff --git a/hack/build-multiarch-operator.sh b/hack/build-multiarch-operator.sh new file mode 100755 index 000000000..21f5857ed --- /dev/null +++ b/hack/build-multiarch-operator.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +check_buildx() { + export DOCKER_CLI_EXPERIMENTAL=enabled + + # If there is no buildx let's install it + if ! docker buildx > /dev/null 2>&1; then + mkdir -p ~/.docker/cli-plugins + curl -L https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 --output ~/.docker/cli-plugins/docker-buildx + chmod a+x ~/.docker/cli-plugins/docker-buildx + fi +} + +create_or_use_buildx_builder() { + local builder_name=$1 + + if [ -z "$builder_name" ]; then + echo "Error: Builder name is required." + exit 1 + fi + + check_buildx + + current_builder="$(docker buildx inspect ${builder_name})" + + # Check if the current builder has multi-architecture support and is not using the "docker" driver + if ! grep -q "^Driver: docker$" <<<"${current_builder}" && \ + grep -q "linux/amd64" <<<"${current_builder}" && \ + grep -q "linux/arm64" <<<"${current_builder}" && \ + grep -q "linux/s390x" <<<"${current_builder}"; then + echo "The current builder already has multi-architecture support (amd64, arm64, s390x)." + echo "Skipping setup as the builder is already configured correctly." + exit 0 + fi + + # Check if the builder already exists by parsing the output of `docker buildx ls` + # We check if the builder_name appears in the list of active builders + existing_builder=$(docker buildx ls | grep -w "$builder_name" | awk '{print $1}') + + if [ -n "$existing_builder" ]; then + echo "Builder '$builder_name' already exists." + echo "Using existing builder '$builder_name'." + docker buildx use "$builder_name" + else + # If the builder does not exist, create a new one + echo "Creating a new Docker Buildx builder: $builder_name" + docker buildx create --driver-opt network=host --use --name "$builder_name" + + # Verify the new builder is set as active + echo "The new builder '$builder_name' has been created and set as active." + fi + +} + +if [ $# -eq 1 ]; then + create_or_use_buildx_builder "$1" +else + echo "Usage: $0 " + echo "Example: $0 mybuilder" + exit 1 +fi