feat: added cronjobs into glasskube repo

devflowinc · Sep 2, 2024 · 839d9f3 · 839d9f3
1 parent adee0f6
commit 839d9f3
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 1 deletion.
diff --git a/glasskube/trieve/configmap.yaml b/glasskube/trieve/configmap.yaml
@@ -16,6 +16,7 @@ data:
   SMTP_PASSWORD: "FROM SECRET"
   SMTP_EMAIL_ADDRESS: "[email protected]"
   OPENAI_API_KEY: "sk-************************************************"
+  ANTHROPIC_API_KEY: "sk-************************************************"
   LLM_API_KEY: "sk-************************************************"
   SECRET_KEY: "01234012340123401234012340123401234012340123401234012340123401234012340123401234" #generate
   SALT: "" # OPTIOANL
@@ -47,7 +48,8 @@ data:
   CLICKHOUSE_DB: "default"
   CLICKHOUSE_USER: "default"
   CLICKHOUSE_PASSWORD: "password"
+  CLICKHOUSE_DSN: "clickhouse://default:password@clickhouse-trieve-clickhouse:8123/default" # Used by Cronjobs
   JINA_CODE_API_KEY: ""
   RUST_LOG: "INFO"
   BM25_ACTIVE: "true"
-  CREATE_QDRANT_COLLECTIONS: "true"
+  CREATE_QDRANT_COLLECTIONS: "true"
diff --git a/glasskube/trieve/cronjobs/clickhouse-collapse.yaml b/glasskube/trieve/cronjobs/clickhouse-collapse.yaml
@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: collapsing-queries-cron
+spec:
+  schedule: "*/30 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: collapsing-queries
+              image: trieve/clickhouse-collapse-query-script
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never
+
diff --git a/glasskube/trieve/cronjobs/cluster-queries.yaml b/glasskube/trieve/cronjobs/cluster-queries.yaml
@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cluster-queries-cronjob
+spec:
+  schedule: "1 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: cluster-queries
+              image: trieve/clickhouse-clustering
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: ANTHROPIC_API_KEY
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: ANTHROPIC_API_KEY
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never
diff --git a/glasskube/trieve/cronjobs/kustomization.yaml b/glasskube/trieve/cronjobs/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+  - word-id-cronjob.yaml
+  - cluster-queries.yaml
+  - clickhouse-collapse.yaml
diff --git a/glasskube/trieve/cronjobs/word-id-cronjob.yaml b/glasskube/trieve/cronjobs/word-id-cronjob.yaml
@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: word-id-cronjob
+  labels:
+    app.kubernetes.io/name: word-id-cronjob
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  schedule: "0 0 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: word-id-cronjob
+              image: trieve/word-id-cronjob
+              ports:
+                - containerPort: 8090
+              env:
+                - name: DATABASE_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-postgresql-server-app
+                      key: uri
+                - name: QDRANT_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-qdrant-qdrant-apikey
+                      key: api-key
+              envFrom:
+                - configMapRef:
+                    name: trieve-server-config
diff --git a/glasskube/trieve/kustomization.yaml b/glasskube/trieve/kustomization.yaml
@@ -11,6 +11,7 @@ resources:
   - ui
   - worker
   - embeddings
+  - cronjobs
 
 images:
   - name: ghcr.io/huggingface/text-embeddings-inference
@@ -38,3 +39,9 @@ images:
   - name: trieve/group-worker
     newName: trieve/group_worker
     newTag: sha-d8b18ca
+  - name: trieve/word-id-cronjob
+    newTag: sha-d8b18ca
+  - name: trieve/clickhouse-collapse-query-script
+    newTag: sha-5cdcd38
+  - name: trieve/clickhouse-clustering
+    newTag: sha-e55860a
diff --git a/glasskube/trieve/ui/configmap.yaml b/glasskube/trieve/ui/configmap.yaml
@@ -8,3 +8,4 @@ data:
   VITE_DASHBOARD_URL: "http://dashboard.localtrieve.com"
   VITE_CHAT_UI_URL: "http://chat.localtrieve.com"
   VITE_ANALYTICS_UI_URL: "http://analytics.localtrieve.com"
+  VITE_BM25_ACTIVE: "true"