From 7124a15ee25df9a8b93e171092271e30616509e0 Mon Sep 17 00:00:00 2001 From: ImpSy <3097030+ImpSy@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:34:31 +0200 Subject: [PATCH] add cicd files for v2 Signed-off-by: ImpSy <3097030+ImpSy@users.noreply.github.com> --- .github/CODEOWNERS | 2 + .github/ocean-spark/Dockerfile | 55 +++++++++ .github/ocean-spark/Makefile | 43 +++++++ .../ocean-spark/actions/release/action.yml | 72 ++++++++++++ .github/workflows/cicd-ofas.yaml | 105 ++++++++++++++++++ 5 files changed, 277 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ocean-spark/Dockerfile create mode 100644 .github/ocean-spark/Makefile create mode 100644 .github/ocean-spark/actions/release/action.yml create mode 100644 .github/workflows/cicd-ofas.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..f1b8b12f3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +* @Spotinst/team-bigdata +/.github/ @Spotinst/team-bigdata-devops diff --git a/.github/ocean-spark/Dockerfile b/.github/ocean-spark/Dockerfile new file mode 100644 index 000000000..c807005e1 --- /dev/null +++ b/.github/ocean-spark/Dockerfile @@ -0,0 +1,55 @@ +# +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG SPARK_IMAGE=spark:3.5.2 + +FROM golang:1.23.1 AS builder + +WORKDIR /workspace + +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=bind,source=go.mod,target=go.mod \ + --mount=type=bind,source=go.sum,target=go.sum \ + go mod download + +COPY . . +ENV GOCACHE=/root/.cache/go-build +ARG TARGETARCH + +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=cache,target="/root/.cache/go-build" \ + CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on make build-operator + +FROM ${SPARK_IMAGE} + +USER root + +RUN yum install -y openssl \ + && yum clean all \ + && curl -fsSL "https://github.com/krallin/tini/releases/download/v0.19.0/tini-static-amd64" -o /sbin/tini \ + && chmod +x /sbin/tini + +RUN mkdir -p /etc/k8s-webhook-server/serving-certs /home/spark && \ + chmod -R g+rw /etc/k8s-webhook-server/serving-certs && \ + chown -R 185 /etc/k8s-webhook-server/serving-certs /home/spark + +USER 185 + +COPY --from=builder /workspace/bin/spark-operator /usr/bin/spark-operator + +COPY entrypoint.sh /usr/bin/ + +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/.github/ocean-spark/Makefile b/.github/ocean-spark/Makefile new file mode 100644 index 000000000..ca8839a09 --- /dev/null +++ b/.github/ocean-spark/Makefile @@ -0,0 +1,43 @@ +REGISTRY ?= +BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) + +VERSION = $(shell git rev-parse --short=7 HEAD) +IMAGE_NAME = spark-operator +SPARK_BASE_IMAGE = public.ecr.aws/f4k1p1n4/spark:netapp-spark-support-image-v3.5-rel20240916 +TAG = $(VERSION) +PLATFORM ?= amd64 + + +build: + docker buildx build --load \ + -t $(REGISTRY)$(IMAGE_NAME):$(TAG_RH)-$(PLATFORM) \ + -f .github/ocean-spark/Dockerfile.rh \ + --platform linux/$(PLATFORM) \ + --build-arg SPARK_IMAGE=$(SPARK_BASE_IMAGE) . + +push: + docker tag $(REGISTRY)$(IMAGE_NAME):$(TAG)-$(PLATFORM) $(REGISTRY)$(IMAGE_NAME):${BRANCH}-$(PLATFORM) + docker push $(REGISTRY)$(IMAGE_NAME):${BRANCH}-$(PLATFORM) + @if [ "$(BRANCH)" = "ocean-spark-v2" ]; then \ + docker push $(REGISTRY)$(IMAGE_NAME):$(TAG)-$(PLATFORM); \ + fi + + +push-manifest: + docker manifest create $(REGISTRY)$(IMAGE_NAME):${BRANCH} \ + $(REGISTRY)$(IMAGE_NAME):${BRANCH}-amd64 \ + $(REGISTRY)$(IMAGE_NAME):${BRANCH}-arm64 + docker manifest push $(REGISTRY)$(IMAGE_NAME):${BRANCH} + @if [ "$(BRANCH)" == "ocean-spark-v2" ]; then \ + docker manifest create $(REGISTRY)$(IMAGE_NAME):$(TAG) \ + $(REGISTRY)$(IMAGE_NAME):$(TAG)-amd64 \ + $(REGISTRY)$(IMAGE_NAME):$(TAG)-arm64; \ + docker manifest push $(REGISTRY)$(IMAGE_NAME):$(TAG); \ + fi + +show: + @echo "Images available:" + @if [ "$(BRANCH)" = "ocean-spark-v2" ]; then \ + echo "-> $(REGISTRY)$(IMAGE_NAME):$(TAG)"; \ + fi + @echo "-> $(REGISTRY)$(IMAGE_NAME):${BRANCH}" \ No newline at end of file diff --git a/.github/ocean-spark/actions/release/action.yml b/.github/ocean-spark/actions/release/action.yml new file mode 100644 index 000000000..21c365522 --- /dev/null +++ b/.github/ocean-spark/actions/release/action.yml @@ -0,0 +1,72 @@ +name: 'Release Action' +description: 'Build and Push the application docker image' + +inputs: + aws-access-key-id: + description: 'AWS Access Key ID' + required: true + aws-secret-access-key: + description: 'AWS Secret Access Key' + required: true + public-registry-id: + description: 'Public ECR Registry ID' + required: true + +runs: + using: "composite" + steps: + - name: Get branch names + id: branch-name + uses: tj-actions/branch-names@v5 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Amazon ECR + uses: docker/login-action@v1 + with: + registry: public.ecr.aws + username: ${{ inputs.aws-access-key-id }} + password: ${{ inputs.aws-secret-access-key }} + + - name: Build AMD64 images + run: make -f .github/ocean-spark/Makefile build + shell: bash + env: + BRANCH: ${{ steps.branch-name.outputs.current_branch }} + REGISTRY: public.ecr.aws/${{ inputs.public-registry-id }}/ + PLATFORM: amd64 + + - name: Push AMD64 images + run: make -f .github/ocean-spark/Makefile push + shell: bash + env: + BRANCH: ${{ steps.branch-name.outputs.current_branch }} + REGISTRY: public.ecr.aws/${{ inputs.public-registry-id }}/ + PLATFORM: amd64 + + - name: Build ARM64 images + run: make -f .github/ocean-spark/Makefile build + shell: bash + env: + BRANCH: ${{ steps.branch-name.outputs.current_branch }} + REGISTRY: public.ecr.aws/${{ inputs.public-registry-id }}/ + PLATFORM: arm64 + + - name: Push ARM64 images + run: make -f .github/ocean-spark/Makefile push + shell: bash + env: + BRANCH: ${{ steps.branch-name.outputs.current_branch }} + REGISTRY: public.ecr.aws/${{ inputs.public-registry-id }}/ + PLATFORM: arm64 + + - name: Push manifest + run: make -f .github/ocean-spark/Makefile push-manifest + shell: bash + env: + BRANCH: ${{ steps.branch-name.outputs.current_branch }} + REGISTRY: public.ecr.aws/${{ inputs.public-registry-id }}/ diff --git a/.github/workflows/cicd-ofas.yaml b/.github/workflows/cicd-ofas.yaml new file mode 100644 index 000000000..f616661f5 --- /dev/null +++ b/.github/workflows/cicd-ofas.yaml @@ -0,0 +1,105 @@ +name: CICD +on: + push: + branches: + - ocean-spark-v2 + pull_request: + branches: + - ocean-spark-v2 + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run go mod tidy + run: | + go mod tidy + if ! git diff --quiet; then + echo "Please run 'go mod tidy' and commit the changes." + git diff + false + fi + + - name: Generate code + run: | + make generate + if ! git diff --quiet; then + echo "Need to re-run 'make generate' and commit the changes." + git diff + false + fi + + - name: Run go fmt check + run: | + make go-fmt + if ! git diff --quiet; then + echo "Need to re-run 'make go-fmt' and commit the changes." + git diff + false + fi + + - name: Run go vet check + run: | + make go-vet + if ! git diff --quiet; then + echo "Need to re-run 'make go-vet' and commit the changes." + git diff + false + fi + + - name: Run golangci-lint + run: | + make go-lint + + - name: Run unit tests + run: make unit-test + + cd-dev-archi: + needs: [ci] + runs-on: ubuntu-latest + environment: dev + + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - if: ${{ github.event_name != 'pull_request' }} + uses: actions/checkout@v4 + + - if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: release to dev + uses: ./.github/ocean-spark/actions/release + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + public-registry-id: n8e8v3t5 + + cd-prod: + if: ${{ github.event_name != 'pull_request' }} + needs: [cd-dev] + runs-on: ubuntu-latest + environment: prod + steps: + - uses: actions/checkout@v4 + + - name: release to prod + uses: ./.github/ocean-spark/actions/release + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + public-registry-id: f4k1p1n4