From 726d653f4206872ad6a9692c2570821fcc41fe1d Mon Sep 17 00:00:00 2001
From: cdxker <fdenzell@gmail.com>
Date: Fri, 30 Aug 2024 15:49:16 -0700
Subject: [PATCH] feat: added cronjobs into glasskube repo

---
 glasskube/trieve/configmap.yaml               |  4 ++-
 .../trieve/cronjobs/clickhouse-collapse.yaml  | 23 +++++++++++++
 .../trieve/cronjobs/cluster-queries.yaml      | 27 +++++++++++++++
 glasskube/trieve/cronjobs/kustomization.yaml  |  4 +++
 .../trieve/cronjobs/word-id-cronjob.yaml      | 34 +++++++++++++++++++
 glasskube/trieve/kustomization.yaml           |  7 ++++
 6 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 glasskube/trieve/cronjobs/clickhouse-collapse.yaml
 create mode 100644 glasskube/trieve/cronjobs/cluster-queries.yaml
 create mode 100644 glasskube/trieve/cronjobs/kustomization.yaml
 create mode 100644 glasskube/trieve/cronjobs/word-id-cronjob.yaml

diff --git a/glasskube/trieve/configmap.yaml b/glasskube/trieve/configmap.yaml
index 24c96caaba..25e10c55a4 100644
--- a/glasskube/trieve/configmap.yaml
+++ b/glasskube/trieve/configmap.yaml
@@ -16,6 +16,7 @@ data:
   SMTP_PASSWORD: "FROM SECRET"
   SMTP_EMAIL_ADDRESS: "noreply@trieve.ai"
   OPENAI_API_KEY: "sk-************************************************"
+  ANTHROPIC_API_KEY: "sk-************************************************"
   LLM_API_KEY: "sk-************************************************"
   SECRET_KEY: "01234012340123401234012340123401234012340123401234012340123401234012340123401234" #generate
   SALT: "" # OPTIOANL
@@ -47,7 +48,8 @@ data:
   CLICKHOUSE_DB: "default"
   CLICKHOUSE_USER: "default"
   CLICKHOUSE_PASSWORD: "password"
+  CLICKHOUSE_DSN: "clickhouse://default:password@clickhouse-trieve-clickhouse:8123/default" # Used by Cronjobs
   JINA_CODE_API_KEY: ""
   RUST_LOG: "INFO"
   BM25_ACTIVE: "true"
-  CREATE_QDRANT_COLLECTIONS: "true"
\ No newline at end of file
+  CREATE_QDRANT_COLLECTIONS: "true"
diff --git a/glasskube/trieve/cronjobs/clickhouse-collapse.yaml b/glasskube/trieve/cronjobs/clickhouse-collapse.yaml
new file mode 100644
index 0000000000..fcd2068b57
--- /dev/null
+++ b/glasskube/trieve/cronjobs/clickhouse-collapse.yaml
@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: collapsing-queries-cron
+spec:
+  schedule: "*/30 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: collapsing-queries
+              image: trieve/clickhouse-collapse-query-script
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never
+
diff --git a/glasskube/trieve/cronjobs/cluster-queries.yaml b/glasskube/trieve/cronjobs/cluster-queries.yaml
new file mode 100644
index 0000000000..440edcd273
--- /dev/null
+++ b/glasskube/trieve/cronjobs/cluster-queries.yaml
@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cluster-queries-cronjob
+spec:
+  schedule: "1 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: cluster-queries
+              image: trieve/clickhouse-clustering
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: ANTHROPIC_API_KEY
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: ANTHROPIC_API_KEY
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never
diff --git a/glasskube/trieve/cronjobs/kustomization.yaml b/glasskube/trieve/cronjobs/kustomization.yaml
new file mode 100644
index 0000000000..a1c5c86f44
--- /dev/null
+++ b/glasskube/trieve/cronjobs/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+  - word-id-cronjob.yaml
+  - cluster-queries.yaml
+  - clickhouse-collapse.yaml
diff --git a/glasskube/trieve/cronjobs/word-id-cronjob.yaml b/glasskube/trieve/cronjobs/word-id-cronjob.yaml
new file mode 100644
index 0000000000..aa450c4e29
--- /dev/null
+++ b/glasskube/trieve/cronjobs/word-id-cronjob.yaml
@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: word-id-cronjob
+  labels:
+    app.kubernetes.io/name: word-id-cronjob
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  schedule: "0 0 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: word-id-cronjob
+              image: trieve/word-id-cronjob
+              ports:
+                - containerPort: 8090
+              env:
+                - name: DATABASE_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-postgresql-server-app
+                      key: uri
+                - name: QDRANT_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-qdrant-qdrant-apikey
+                      key: api-key
+              envFrom:
+                - configMapRef:
+                    name: trieve-server-config
diff --git a/glasskube/trieve/kustomization.yaml b/glasskube/trieve/kustomization.yaml
index c1264b0d22..0f605ac47b 100644
--- a/glasskube/trieve/kustomization.yaml
+++ b/glasskube/trieve/kustomization.yaml
@@ -11,6 +11,7 @@ resources:
   - ui
   - worker
   - embeddings
+  - cronjobs
 
 images:
   - name: ghcr.io/huggingface/text-embeddings-inference
@@ -38,3 +39,9 @@ images:
   - name: trieve/group-worker
     newName: trieve/group_worker
     newTag: sha-d8b18ca
+  - name: trieve/word-id-cronjob
+    newTag: sha-d8b18ca
+  - name: trieve/clickhouse-collapse-query-script
+    newTag: sha-5cdcd38
+  - name: trieve/clickhouse-clustering
+    newTag: sha-e55860a