chore: Jmeter build into action (#113)

* build: add jmeter build to workflow

* build: move the base image to artifact repo

* chore: a simple script for moving the images from old gcr registry to new artifact repo

* fix: move the auth file to secret

* fix: move the engine agent config into configmap

* chore: update some comments to explain the logic

* fix: deal with nil config

* fix: running image should have ca

.github/workflows/build-publish.yml

@@ -102,6 +102,13 @@ jobs:
       run: |-
         cd shibuya && make controller_image component=controller
 
+    - name: build jmeter agent
+      env:
+        tag_name: ${{ needs.release-please.outputs.tag_name }}
+      # we specify the jmeter version we are using in the component name
+      # So we can cleary know which jmeter version we are using
+      run: |-
+        cd shibuya && make jmeter_agent_image component=jmeter-3.3
 
 
 

makefile

@@ -52,9 +52,8 @@ shibuya: local_api local_controller
 
 .PHONY: jmeter
 jmeter: shibuya/engines/jmeter
-	cp shibuya/config_tmpl.json shibuya/config.json
 	cd shibuya && sh build.sh jmeter
-	docker build -t shibuya:jmeter -f shibuya/docker-local/Dockerfile.engines.jmeter shibuya
+	docker build -t shibuya:jmeter -f shibuya/Dockerfile.engines.jmeter shibuya
 	kind load docker-image shibuya:jmeter --name shibuya
 
 .PHONY: expose

shibuya/Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:18.04
 
+RUN apt-get update && apt-get install -y ca-certificates
 ARG binary_name=shibuya
 ADD ./build/${binary_name} /usr/local/bin/${binary_name}
 

shibuya/Dockerfile.engines.jmeter

@@ -1,31 +1,18 @@
 ARG jmeter_ver=3.3
 
-FROM gcr.io/shibuya-214807/alpine:3.10.2 AS jmeter
+FROM asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya/alpine:3.10.2 AS jmeter
 ARG jmeter_ver
 ENV JMETER_VERSION=$jmeter_ver
 RUN wget archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip
 RUN unzip -qq apache-jmeter-${JMETER_VERSION}
 
-FROM gcr.io/shibuya-214807/golang:1.13.6-stretch AS shibuya-agent
-WORKDIR /go/src/shibuya
-ENV GO111MODULE on
-ADD go.mod .
-ADD go.sum .
-RUN go mod download
-
-COPY . /go/src/shibuya
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /go/bin/shibuya-agent /go/src/shibuya/engines/jmeter
-
-FROM gcr.io/shibuya-214807/openjdk:8u212-jdk
+FROM asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya/openjdk:8u212-jdk
 ARG jmeter_ver
 ENV JMETER_VERSION=$jmeter_ver
 RUN mkdir /test-conf /test-result
 COPY --from=jmeter /apache-jmeter-${JMETER_VERSION} /apache-jmeter-${JMETER_VERSION}
-COPY --from=shibuya-agent /go/bin/shibuya-agent /usr/local/bin/shibuya-agent
-COPY config.json config.json
+ADD build/shibuya-agent /usr/local/bin/shibuya-agent
 ADD engines/jmeter/shibuya.properties /test-conf/shibuya.properties
 ADD engines/jmeter/jmeter.sh /apache-jmeter-${JMETER_VERSION}/bin/jmeter
-RUN mkdir /auth
-ADD ./shibuya-gcp.json /auth/shibuya-gcp.json
-ENV GOOGLE_APPLICATION_CREDENTIALS /auth/shibuya-gcp.json
-CMD ["shibuya-agent"]
+
+CMD ["shibuya-agent"]

shibuya/Makefile

@@ -31,3 +31,14 @@ controller_image: controller_build
 .PHONY: helm_charts
 helm_charts:
 	helm package install/shibuya
+
+.PHONY: jmeter_agent
+jmeter_agent:
+	sh build.sh jmeter
+
+.PHONY: jmeter_agent_image
+jmeter_agent_image: jmeter_agent
+	docker build -t $(img) -f Dockerfile.engines.jmeter .
+	docker push $(img)
+
+

shibuya/config/init.go

@@ -13,6 +13,14 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 )
 
+const (
+	ConfigFileName = "config.json"
+)
+
+var (
+	ConfigFilePath = path.Join("/", ConfigFileName)
+)
+
 type LdapConfig struct {
 	BaseDN         string `json:"base_dn"`
 	SystemUser     string `json:"system_user"`
@@ -92,6 +100,12 @@ type ObjectStorage struct {
 	Password     string `json:"password"`
 	Bucket       string `json:"bucket"`
 	RequireProxy bool   `json:"require_proxy"`
+	// This is the secret name created in the cluster for authenticating with object storage
+	SecretName string `json:"secret_name"`
+	// This is the mounted keys file name. e.g. /auth/shibuya-gcp.json
+	AuthFileName string `json:"auth_file_name"`
+	// This is the configuration file
+	ConfigMapName string `json:"config_map_name"`
 }
 
 type LogFormat struct {
@@ -178,15 +192,17 @@ func applyJsonLogging() {
 func setupLogging() {
 	log.SetOutput(os.Stdout)
 	log.SetReportCaller(true)
-	if SC.LogFormat.Json {
-		applyJsonLogging()
+	if SC.LogFormat != nil {
+		if SC.LogFormat.Json {
+			applyJsonLogging()
+		}
 	}
 }
 
 func loadConfig() *ShibuyaConfig {
 	sc := new(ShibuyaConfig)
 	sc.IngressConfig = &defaultIngressConfig
-	f, err := os.Open("/config.json")
+	f, err := os.Open(ConfigFilePath)
 	if err != nil {
 		log.Fatal("Cannot find config file")
 	}
@@ -199,8 +215,9 @@ func loadConfig() *ShibuyaConfig {
 	}
 	sc.Context = loadContext()
 	sc.DevMode = sc.Context == "local"
-	sc.makeHTTPClients()
-
+	if sc.HttpConfig != nil {
+		sc.makeHTTPClients()
+	}
 	// In jmeter agent, we also rely on this module, therefore we need to check whether this is nil or not. As jmeter
 	// configuration might provide an empty struct here
 	// TODO: we should not let jmeter code rely on this part

shibuya/install/shibuya/templates/configmap.yaml

@@ -86,7 +86,10 @@ data:
             {{- with .Values.runtime.object_storage.url }}
             "url": {{ . | quote }},
             {{- end }}
-            "bucket": {{ .Values.runtime.object_storage.bucket | quote }}
+            "bucket": {{ .Values.runtime.object_storage.bucket | quote }},
+            "secret_name": {{ .Values.runtime.object_storage.secret_name | quote }},
+            "auth_file_name": {{ .Values.runtime.object_storage.auth_file_name | quote }},
+            "config_map_name": {{ .Values.runtime.object_storage.config_map_name | quote }}
         },
         "log_format": {
             "json": false

shibuya/install/shibuya/values.yaml

@@ -97,3 +97,6 @@ runtime:
     user: ""
     password: ""
     bucket: ""
+    secret_name: ""
+    auth_file_name: ""
+    config_map_name: "shibuya-config-local"

shibuya/move_images.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+new_registry=asia-northeast1-docker.pkg.dev/shibuya-214807/shibuya
+old_registry=gcr.io/shibuya-214807
+
+component=$1
+old_image=$old_registry/$component
+new_image=$new_registry/$component
+docker pull $old_image
+docker tag $old_image $new_image
+docker push $new_image

shibuya/object_storage/factory.go

@@ -47,6 +47,10 @@ func factoryConfig() PlatformConfig {
 	}
 }
 
+func IsProviderGCP() bool {
+	return config.SC.ObjectStorage.Provider == gcpStorageProvider
+}
+
 var Client PlatformConfig
 
 func init() {

shibuya/scheduler/k8s.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/rakutentech/shibuya/shibuya/config"
 	model "github.com/rakutentech/shibuya/shibuya/model"
+	"github.com/rakutentech/shibuya/shibuya/object_storage"
 	smodel "github.com/rakutentech/shibuya/shibuya/scheduler/model"
 	log "github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
@@ -150,6 +151,54 @@ func (kcm *K8sClientManager) makeHostAliases() []apiv1.HostAlias {
 func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas int, labels map[string]string, containerConfig *config.ExecutorContainer,
 	affinity *apiv1.Affinity, tolerations []apiv1.Toleration) appsv1.StatefulSet {
 	t := true
+	volumes := []apiv1.Volume{}
+	volumeMounts := []apiv1.VolumeMount{}
+	envvars := []apiv1.EnvVar{}
+	if object_storage.IsProviderGCP() {
+		volumeName := "shibuya-gcp-auth"
+		secretName := config.SC.ObjectStorage.SecretName
+		authFileName := config.SC.ObjectStorage.AuthFileName
+		mountPath := fmt.Sprintf("/auth/%s", authFileName)
+		v := apiv1.Volume{
+			Name: volumeName,
+			VolumeSource: apiv1.VolumeSource{
+				Secret: &apiv1.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		volumes = append(volumes, v)
+		vm := apiv1.VolumeMount{
+			Name:      volumeName,
+			MountPath: mountPath,
+			SubPath:   authFileName,
+		}
+		volumeMounts = append(volumeMounts, vm)
+		envvar := apiv1.EnvVar{
+			Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+			Value: mountPath,
+		}
+		envvars = append(envvars, envvar)
+	}
+	cmVolumeName := "shibuya-config"
+	cmName := config.SC.ObjectStorage.ConfigMapName
+	cmVolume := apiv1.Volume{
+		Name: cmVolumeName,
+		VolumeSource: apiv1.VolumeSource{
+			ConfigMap: &apiv1.ConfigMapVolumeSource{
+				LocalObjectReference: apiv1.LocalObjectReference{
+					Name: cmName,
+				},
+			},
+		},
+	}
+	volumes = append(volumes, cmVolume)
+	cmVolumeMounts := apiv1.VolumeMount{
+		Name:      cmVolumeName,
+		MountPath: config.ConfigFilePath,
+		SubPath:   config.ConfigFileName,
+	}
+	volumeMounts = append(volumeMounts, cmVolumeMounts)
 	deployment := appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:                       planName,
@@ -178,6 +227,7 @@ func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas in
 					},
 					TerminationGracePeriodSeconds: new(int64),
 					HostAliases:                   kcm.makeHostAliases(),
+					Volumes:                       volumes,
 					Containers: []apiv1.Container{
 						{
 							Name:            planName,
@@ -200,6 +250,8 @@ func (kcm *K8sClientManager) generatePlanDeployment(planName string, replicas in
 									ContainerPort: 8080,
 								},
 							},
+							VolumeMounts: volumeMounts,
+							Env:          envvars,
 						},
 					},
 				},