remove minio gateway (#950)

neuro-inc · Aug 9, 2024 · 904be33 · 904be33
1 parent 5a94149
commit 904be33
Show file tree

Hide file tree

Showing 13 changed files with 56 additions and 361 deletions.
diff --git a/charts/platform-monitoring/files/fluent-bit/fluent-bit.conf b/charts/platform-monitoring/files/fluent-bit/fluent-bit.conf
@@ -49,25 +49,11 @@
 [OUTPUT]
     Name             s3
     Match            kube.var.log.containers.job-*
-{{- if eq .Values.logs.persistence.type "aws" }}
-{{- if .Values.logs.persistence.aws.endpoint }}
-    endpoint         {{ .Values.logs.persistence.aws.endpoint }}
-{{- end }}
-    region           {{ .Values.logs.persistence.aws.region }}
-    bucket           {{ .Values.logs.persistence.aws.bucket }}
-{{- else if eq .Values.logs.persistence.type "minio" }}
-    endpoint         {{ .Values.logs.persistence.minio.url }}
-    region           {{ .Values.logs.persistence.minio.region }}
-    bucket           {{ .Values.logs.persistence.minio.bucket }}
-{{- else if eq .Values.logs.persistence.type "gcp" }}
-    endpoint         {{ include "platformMonitoring.minioGateway.endpoint" . }}
-    region           {{ .Values.logs.persistence.gcp.location }}
-    bucket           {{ .Values.logs.persistence.gcp.bucket }}
-{{- else if eq .Values.logs.persistence.type "azure" }}
-    endpoint         {{ include "platformMonitoring.minioGateway.endpoint" . }}
-    region           minio
-    bucket           {{ .Values.logs.persistence.azure.bucket }}
+{{- if .Values.logs.persistence.s3.endpoint }}
+    endpoint         {{ .Values.logs.persistence.s3.endpoint }}
 {{- end }}
+    region           {{ .Values.logs.persistence.s3.region }}
+    bucket           {{ .Values.logs.persistence.s3.bucket }}
     total_file_size  1M
     upload_timeout   1m
     use_put_object   On

diff --git a/charts/platform-monitoring/templates/_helpers.tpl b/charts/platform-monitoring/templates/_helpers.tpl
@@ -28,24 +28,6 @@
 {{- end -}}
 {{- end -}}
 
-{{- define "platformMonitoring.minioGateway.fullname" -}}
-{{- if .Values.minioGateway.fullnameOverride -}}
-{{- .Values.minioGateway.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default "minio-gateway" .Values.minioGateway.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "platformMonitoring.minioGateway.endpoint" -}}
-{{- $serviceName := include "platformMonitoring.minioGateway.fullname" . -}}
-{{- printf "http://%s:%s" $serviceName (toString .Values.minioGateway.port) -}}
-{{- end -}}
-
 {{- define "platformMonitoring.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" -}}
 {{- end -}}
@@ -57,14 +39,6 @@ heritage: {{ .Release.Service | quote }}
 release: {{ .Release.Name | quote }}
 {{- end -}}
 
-{{- define "platformMonitoring.logs.storage.keySecret" -}}
-{{- if .Values.logs.persistence.keySecret -}}
-{{ .Values.logs.persistence.keySecret }}
-{{- else -}}
-{{ include "platformMonitoring.fullname" . }}-logs-storage-key
-{{- end -}}
-{{- end -}}
-
 {{- define "platformMonitoring.kubeAuthMountRoot" -}}
 {{- printf "/var/run/secrets/kubernetes.io/serviceaccount" -}}
 {{- end -}}
@@ -130,80 +104,27 @@ release: {{ .Release.Name | quote }}
 
 {{- define "platformMonitoring.env.s3" -}}
 {{- $logsPersistence := .Values.logs.persistence -}}
-{{- if eq $logsPersistence.type "aws" }}
+{{- if eq $logsPersistence.type "s3" }}
 - name: NP_MONITORING_LOGS_STORAGE_TYPE
   value: s3
+{{- if $logsPersistence.s3.accessKeyId }}
 - name: NP_MONITORING_S3_ACCESS_KEY_ID
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: access_key_id
+  {{- toYaml $logsPersistence.s3.accessKeyId | nindent 2 }}
+{{- end }}
+{{- if $logsPersistence.s3.secretAccessKey }}
 - name: NP_MONITORING_S3_SECRET_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: secret_access_key
-{{- if $logsPersistence.aws.region }}
+  {{- toYaml $logsPersistence.s3.secretAccessKey | nindent 2 }}
+{{- end }}
+{{- if $logsPersistence.s3.region }}
 - name: NP_MONITORING_S3_REGION
-  value: {{ $logsPersistence.aws.region | quote }}
+  value: {{ $logsPersistence.s3.region | quote }}
 {{- end }}
-{{- if $logsPersistence.aws.endpoint }}
+{{- if $logsPersistence.s3.endpoint }}
 - name: NP_MONITORING_S3_ENDPOINT_URL
-  value: {{ $logsPersistence.aws.endpoint | quote }}
+  value: {{ $logsPersistence.s3.endpoint | quote }}
 {{- end }}
 - name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
-  value: {{ $logsPersistence.aws.bucket | quote }}
-{{- else if eq $logsPersistence.type "gcp" }}
-- name: NP_MONITORING_LOGS_STORAGE_TYPE
-  value: s3
-- name: NP_MONITORING_S3_REGION
-  value: {{ $logsPersistence.gcp.location | quote }}
-- name: NP_MONITORING_S3_ACCESS_KEY_ID
-  value: minio_access_key
-- name: NP_MONITORING_S3_SECRET_ACCESS_KEY
-  value: minio_secret_key
-- name: NP_MONITORING_S3_ENDPOINT_URL
-  value: {{ include "platformMonitoring.minioGateway.endpoint" . }}
-- name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
-  value: {{ $logsPersistence.gcp.bucket | quote }}
-{{- else if eq $logsPersistence.type "azure" }}
-- name: NP_MONITORING_LOGS_STORAGE_TYPE
-  value: s3
-- name: NP_MONITORING_S3_REGION
-  value: minio
-- name: NP_MONITORING_S3_ACCESS_KEY_ID
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: account_name
-- name: NP_MONITORING_S3_SECRET_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: account_key
-- name: NP_MONITORING_S3_ENDPOINT_URL
-  value: {{ include "platformMonitoring.minioGateway.endpoint" . }}
-- name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
-  value: {{ $logsPersistence.azure.bucket | quote }}
-{{- else if eq $logsPersistence.type "minio" }}
-- name: NP_MONITORING_LOGS_STORAGE_TYPE
-  value: s3
-- name: NP_MONITORING_S3_ENDPOINT_URL
-  value: {{ $logsPersistence.minio.url | quote }}
-- name: NP_MONITORING_S3_REGION
-  value: {{ $logsPersistence.minio.region | quote }}
-- name: NP_MONITORING_S3_ACCESS_KEY_ID
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: access_key
-- name: NP_MONITORING_S3_SECRET_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      key: secret_key
-- name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
-  value: {{ $logsPersistence.minio.bucket | quote }}
+  value: {{ $logsPersistence.s3.bucket | quote }}
 {{- end }}
 {{- end -}}
 

diff --git a/charts/platform-monitoring/templates/deployment.yml b/charts/platform-monitoring/templates/deployment.yml
@@ -24,7 +24,6 @@ spec:
         release: {{ .Release.Name }}
         service: platform-monitoring
       annotations:
-        checksum/logs-secret: {{ include (print $.Template.BasePath "/logs-storage-key-secret.yaml") . | sha256sum }}
         {{- if .Values.secrets }}
         checksum/secret: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         {{- end }}

diff --git a/charts/platform-monitoring/templates/fluent-bit.yaml b/charts/platform-monitoring/templates/fluent-bit.yaml
@@ -44,48 +44,15 @@ spec:
         image: {{ .repository }}:{{ .tag }}
         {{- end }}
         env:
-        {{- if eq .Values.logs.persistence.type "aws" }}
-        {{- if .Values.logs.persistence.aws.accessKeyId }}
+        {{- if eq .Values.logs.persistence.type "s3" }}
+        {{- if .Values.logs.persistence.s3.accessKeyId }}
         - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: access_key_id
+          {{- toYaml .Values.logs.persistence.s3.accessKeyId | nindent 10 }}
         {{- end }}
-        {{- if .Values.logs.persistence.aws.secretAccessKey }}
+        {{- if .Values.logs.persistence.s3.secretAccessKey }}
         - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: secret_access_key
+          {{- toYaml .Values.logs.persistence.s3.secretAccessKey | nindent 10 }}
         {{- end }}
-        {{- else if eq .Values.logs.persistence.type "minio" }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: access_key
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: secret_key
-        {{- else if eq .Values.logs.persistence.type "gcp" }}
-        - name: AWS_ACCESS_KEY_ID
-          value: minio_access_key
-        - name: AWS_SECRET_ACCESS_KEY
-          value: minio_secret_key
-        {{- else if eq .Values.logs.persistence.type "azure" }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: account_name
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: account_key
         {{- end }}
         volumeMounts:
         - mountPath: /var/log

diff --git a/charts/platform-monitoring/templates/logs-compact-deployment.yaml b/charts/platform-monitoring/templates/logs-compact-deployment.yaml
@@ -1,4 +1,4 @@
-{{- if ne .Values.logs.persistence.type "es" }}
+{{- if eq .Values.logs.persistence.type "s3" }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

diff --git a/charts/platform-monitoring/templates/logs-storage-key-secret.yaml b/charts/platform-monitoring/templates/logs-storage-key-secret.yaml
diff --git a/charts/platform-monitoring/templates/minio-gateway.yaml b/charts/platform-monitoring/templates/minio-gateway.yaml