From 9ae15b1b98db58de8d938cdfd315b4a7ff0b8939 Mon Sep 17 00:00:00 2001 From: Andy Huang Date: Thu, 29 Aug 2024 21:09:11 +0900 Subject: [PATCH] chore: Jmeter build into action (#113) * build: add jmeter build to workflow * build: move the base image to artifact repo * chore: a simple script for moving the images from old gcr registry to new artifact repo * fix: move the auth file to secret * fix: move the engine agent config into configmap * chore: update some comments to explain the logic * fix: deal with nil config * fix: running image should have ca --- .github/workflows/build-publish.yml | 7 +++ makefile | 3 +- shibuya/Dockerfile | 1 + shibuya/Dockerfile.engines.jmeter | 23 ++------ shibuya/Makefile | 11 ++++ shibuya/config/init.go | 27 ++++++++-- .../docker-local/Dockerfile.engines.jmeter | 21 -------- shibuya/docker-local/README.md | 9 ---- .../install/shibuya/templates/configmap.yaml | 5 +- shibuya/install/shibuya/values.yaml | 3 ++ shibuya/move_images.sh | 11 ++++ shibuya/object_storage/factory.go | 4 ++ shibuya/scheduler/k8s.go | 52 +++++++++++++++++++ 13 files changed, 121 insertions(+), 56 deletions(-) delete mode 100644 shibuya/docker-local/Dockerfile.engines.jmeter delete mode 100644 shibuya/docker-local/README.md create mode 100644 shibuya/move_images.sh diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml index f2e5e3b4..22ad32a7 100644 --- a/.github/workflows/build-publish.yml +++ b/.github/workflows/build-publish.yml @@ -102,6 +102,13 @@ jobs: run: |- cd shibuya && make controller_image component=controller + - name: build jmeter agent + env: + tag_name: ${{ needs.release-please.outputs.tag_name }} + # we specify the jmeter version we are using in the component name + # So we can cleary know which jmeter version we are using + run: |- + cd shibuya && make jmeter_agent_image component=jmeter-3.3 diff --git a/makefile b/makefile index 3923dde1..e471ca65 100644 --- a/makefile +++ b/makefile @@ -52,9 +52,8 @@ shibuya: local_api local_controller .PHONY: jmeter jmeter: shibuya/engines/jmeter - cp shibuya/config_tmpl.json shibuya/config.json cd shibuya && sh build.sh jmeter - docker build -t shibuya:jmeter -f shibuya/docker-local/Dockerfile.engines.jmeter shibuya + docker build -t shibuya:jmeter -f shibuya/Dockerfile.engines.jmeter shibuya kind load docker-image shibuya:jmeter --name shibuya .PHONY: expose diff --git a/shibuya/Dockerfile b/shibuya/Dockerfile index 7642b11e..0865972b 100644 --- a/shibuya/Dockerfile +++ b/shibuya/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:18.04 +RUN apt-get update && apt-get install -y ca-certificates ARG binary_name=shibuya ADD ./build/${binary_name} /usr/local/bin/${binary_name} diff --git a/shibuya/Dockerfile.engines.jmeter b/shibuya/Dockerfile.engines.jmeter index 7ac35824..e17deec1 100644 --- a/shibuya/Dockerfile.engines.jmeter +++ b/shibuya/Dockerfile.engines.jmeter @@ -1,31 +1,18 @@ ARG jmeter_ver=3.3 -FROM gcr.io/shibuya-214807/alpine:3.10.2 AS jmeter +FROM asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya/alpine:3.10.2 AS jmeter ARG jmeter_ver ENV JMETER_VERSION=$jmeter_ver RUN wget archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip RUN unzip -qq apache-jmeter-${JMETER_VERSION} -FROM gcr.io/shibuya-214807/golang:1.13.6-stretch AS shibuya-agent -WORKDIR /go/src/shibuya -ENV GO111MODULE on -ADD go.mod . -ADD go.sum . -RUN go mod download - -COPY . /go/src/shibuya -RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /go/bin/shibuya-agent /go/src/shibuya/engines/jmeter - -FROM gcr.io/shibuya-214807/openjdk:8u212-jdk +FROM asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya/openjdk:8u212-jdk ARG jmeter_ver ENV JMETER_VERSION=$jmeter_ver RUN mkdir /test-conf /test-result COPY --from=jmeter /apache-jmeter-${JMETER_VERSION} /apache-jmeter-${JMETER_VERSION} -COPY --from=shibuya-agent /go/bin/shibuya-agent /usr/local/bin/shibuya-agent -COPY config.json config.json +ADD build/shibuya-agent /usr/local/bin/shibuya-agent ADD engines/jmeter/shibuya.properties /test-conf/shibuya.properties ADD engines/jmeter/jmeter.sh /apache-jmeter-${JMETER_VERSION}/bin/jmeter -RUN mkdir /auth -ADD ./shibuya-gcp.json /auth/shibuya-gcp.json -ENV GOOGLE_APPLICATION_CREDENTIALS /auth/shibuya-gcp.json -CMD ["shibuya-agent"] \ No newline at end of file + +CMD ["shibuya-agent"] diff --git a/shibuya/Makefile b/shibuya/Makefile index f2915669..e1b85af6 100644 --- a/shibuya/Makefile +++ b/shibuya/Makefile @@ -31,3 +31,14 @@ controller_image: controller_build .PHONY: helm_charts helm_charts: helm package install/shibuya + +.PHONY: jmeter_agent +jmeter_agent: + sh build.sh jmeter + +.PHONY: jmeter_agent_image +jmeter_agent_image: jmeter_agent + docker build -t $(img) -f Dockerfile.engines.jmeter . + docker push $(img) + + diff --git a/shibuya/config/init.go b/shibuya/config/init.go index 802e8741..c7bad19f 100644 --- a/shibuya/config/init.go +++ b/shibuya/config/init.go @@ -13,6 +13,14 @@ import ( apiv1 "k8s.io/api/core/v1" ) +const ( + ConfigFileName = "config.json" +) + +var ( + ConfigFilePath = path.Join("/", ConfigFileName) +) + type LdapConfig struct { BaseDN string `json:"base_dn"` SystemUser string `json:"system_user"` @@ -92,6 +100,12 @@ type ObjectStorage struct { Password string `json:"password"` Bucket string `json:"bucket"` RequireProxy bool `json:"require_proxy"` + // This is the secret name created in the cluster for authenticating with object storage + SecretName string `json:"secret_name"` + // This is the mounted keys file name. e.g. /auth/shibuya-gcp.json + AuthFileName string `json:"auth_file_name"` + // This is the configuration file + ConfigMapName string `json:"config_map_name"` } type LogFormat struct { @@ -178,15 +192,17 @@ func applyJsonLogging() { func setupLogging() { log.SetOutput(os.Stdout) log.SetReportCaller(true) - if SC.LogFormat.Json { - applyJsonLogging() + if SC.LogFormat != nil { + if SC.LogFormat.Json { + applyJsonLogging() + } } } func loadConfig() *ShibuyaConfig { sc := new(ShibuyaConfig) sc.IngressConfig = &defaultIngressConfig - f, err := os.Open("/config.json") + f, err := os.Open(ConfigFilePath) if err != nil { log.Fatal("Cannot find config file") } @@ -199,8 +215,9 @@ func loadConfig() *ShibuyaConfig { } sc.Context = loadContext() sc.DevMode = sc.Context == "local" - sc.makeHTTPClients() - + if sc.HttpConfig != nil { + sc.makeHTTPClients() + } // In jmeter agent, we also rely on this module, therefore we need to check whether this is nil or not. As jmeter // configuration might provide an empty struct here // TODO: we should not let jmeter code rely on this part diff --git a/shibuya/docker-local/Dockerfile.engines.jmeter b/shibuya/docker-local/Dockerfile.engines.jmeter deleted file mode 100644 index 56391a6d..00000000 --- a/shibuya/docker-local/Dockerfile.engines.jmeter +++ /dev/null @@ -1,21 +0,0 @@ -ARG jmeter_ver=3.3 - -FROM gcr.io/shibuya-214807/alpine:3.10.2 AS jmeter -ARG jmeter_ver -ENV JMETER_VERSION=$jmeter_ver -RUN wget archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip -RUN unzip -qq apache-jmeter-${JMETER_VERSION} - -FROM gcr.io/shibuya-214807/openjdk:8u212-jdk -ARG jmeter_ver -ENV JMETER_VERSION=$jmeter_ver -RUN mkdir /test-conf /test-result -COPY --from=jmeter /apache-jmeter-${JMETER_VERSION} /apache-jmeter-${JMETER_VERSION} -ADD ./build/shibuya-agent /usr/local/bin/shibuya-agent -COPY config.json config.json -ADD engines/jmeter/shibuya.properties /test-conf/shibuya.properties -ADD engines/jmeter/jmeter.sh /apache-jmeter-${JMETER_VERSION}/bin/jmeter -RUN mkdir /auth -ADD ./shibuya-gcp.json /auth/shibuya-gcp.json -ENV GOOGLE_APPLICATION_CREDENTIALS /auth/shibuya-gcp.json -CMD ["shibuya-agent"] \ No newline at end of file diff --git a/shibuya/docker-local/README.md b/shibuya/docker-local/README.md deleted file mode 100644 index a50db680..00000000 --- a/shibuya/docker-local/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Why we need a separate files for building images - -It's mysteriously slow on my local Mac for building Go binaries that are depending on k8s libaries in Docker containers. As a result, in order to fastly create local environment, I moved the building process into Mac utilising Go's cross-platform compilation capability. - -On the other hand, I could not break the exsiting Dockerfiles as they are required for CI/CD otherwise it will require our current CI/CD to have Go runtime support. - -# How can we ensure the running environments are the same - -Please remember to update two Dockerfiles when it's necessary. \ No newline at end of file diff --git a/shibuya/install/shibuya/templates/configmap.yaml b/shibuya/install/shibuya/templates/configmap.yaml index 31262b53..14ccb1ca 100644 --- a/shibuya/install/shibuya/templates/configmap.yaml +++ b/shibuya/install/shibuya/templates/configmap.yaml @@ -86,7 +86,10 @@ data: {{- with .Values.runtime.object_storage.url }} "url": {{ . | quote }}, {{- end }} - "bucket": {{ .Values.runtime.object_storage.bucket | quote }} + "bucket": {{ .Values.runtime.object_storage.bucket | quote }}, + "secret_name": {{ .Values.runtime.object_storage.secret_name | quote }}, + "auth_file_name": {{ .Values.runtime.object_storage.auth_file_name | quote }}, + "config_map_name": {{ .Values.runtime.object_storage.config_map_name | quote }} }, "log_format": { "json": false diff --git a/shibuya/install/shibuya/values.yaml b/shibuya/install/shibuya/values.yaml index b1e6d39c..5ace1e3b 100644 --- a/shibuya/install/shibuya/values.yaml +++ b/shibuya/install/shibuya/values.yaml @@ -97,3 +97,6 @@ runtime: user: "" password: "" bucket: "" + secret_name: "" + auth_file_name: "" + config_map_name: "shibuya-config-local" diff --git a/shibuya/move_images.sh b/shibuya/move_images.sh new file mode 100644 index 00000000..36540e67 --- /dev/null +++ b/shibuya/move_images.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +new_registry=asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya +old_registry=gcr.io/shibuya-214807 + +component=$1 +old_image=$old_registry/$component +new_image=$new_registry/$component +docker pull $old_image +docker tag $old_image $new_image +docker push $new_image diff --git a/shibuya/object_storage/factory.go b/shibuya/object_storage/factory.go index de484ed8..d37c7a90 100644 --- a/shibuya/object_storage/factory.go +++ b/shibuya/object_storage/factory.go @@ -47,6 +47,10 @@ func factoryConfig() PlatformConfig { } } +func IsProviderGCP() bool { + return config.SC.ObjectStorage.Provider == gcpStorageProvider +} + var Client PlatformConfig func init() { diff --git a/shibuya/scheduler/k8s.go b/shibuya/scheduler/k8s.go index 5b5b0ef9..e742d650 100644 --- a/shibuya/scheduler/k8s.go +++ b/shibuya/scheduler/k8s.go @@ -13,6 +13,7 @@ import ( "github.com/rakutentech/shibuya/shibuya/config" model "github.com/rakutentech/shibuya/shibuya/model" + "github.com/rakutentech/shibuya/shibuya/object_storage" smodel "github.com/rakutentech/shibuya/shibuya/scheduler/model" log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" @@ -150,6 +151,54 @@ func (kcm *K8sClientManager) makeHostAliases() []apiv1.HostAlias { func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas int, labels map[string]string, containerConfig *config.ExecutorContainer, affinity *apiv1.Affinity, tolerations []apiv1.Toleration) appsv1.StatefulSet { t := true + volumes := []apiv1.Volume{} + volumeMounts := []apiv1.VolumeMount{} + envvars := []apiv1.EnvVar{} + if object_storage.IsProviderGCP() { + volumeName := "shibuya-gcp-auth" + secretName := config.SC.ObjectStorage.SecretName + authFileName := config.SC.ObjectStorage.AuthFileName + mountPath := fmt.Sprintf("/auth/%s", authFileName) + v := apiv1.Volume{ + Name: volumeName, + VolumeSource: apiv1.VolumeSource{ + Secret: &apiv1.SecretVolumeSource{ + SecretName: secretName, + }, + }, + } + volumes = append(volumes, v) + vm := apiv1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + SubPath: authFileName, + } + volumeMounts = append(volumeMounts, vm) + envvar := apiv1.EnvVar{ + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: mountPath, + } + envvars = append(envvars, envvar) + } + cmVolumeName := "shibuya-config" + cmName := config.SC.ObjectStorage.ConfigMapName + cmVolume := apiv1.Volume{ + Name: cmVolumeName, + VolumeSource: apiv1.VolumeSource{ + ConfigMap: &apiv1.ConfigMapVolumeSource{ + LocalObjectReference: apiv1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + } + volumes = append(volumes, cmVolume) + cmVolumeMounts := apiv1.VolumeMount{ + Name: cmVolumeName, + MountPath: config.ConfigFilePath, + SubPath: config.ConfigFileName, + } + volumeMounts = append(volumeMounts, cmVolumeMounts) deployment := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: planName, @@ -178,6 +227,7 @@ func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas in }, TerminationGracePeriodSeconds: new(int64), HostAliases: kcm.makeHostAliases(), + Volumes: volumes, Containers: []apiv1.Container{ { Name: planName, @@ -200,6 +250,8 @@ func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas in ContainerPort: 8080, }, }, + VolumeMounts: volumeMounts, + Env: envvars, }, }, },