feat: added more workers, moved ingest to the workers template

.github/workflows/ts-sdk-gh-pages.yml

@@ -2,6 +2,12 @@ name: Deploy TS-SDK docs to github pages
 
 on:
   push:
+    paths:
+    - 'clients/ts-sdk/**'
+    branches: ["main"]
+  pull_request:
+    paths:
+    - 'clients/ts-sdk/**'
     branches: ["main"]
 
 jobs:

glasskube/README.md

@@ -70,9 +70,15 @@ Open [dashboard.localtrieve.com](http://dashboard.localtrieve.com) in your brows
 
 ## Optional commands
 
-Scale down all embedding servers:
+#### Scale down all embedding servers:
 
 ```shell
 kubectl scale deployment trieve-embedding-bgem3  trieve-embedding-jina trieve-embedding-reranker trieve-embedding-splade-doc trieve-embedding-splade-query --replicas 0
 
-```
+```
+
+#### Deleting the cluster
+
+```shell
+minikube delete -p trieve
+```

glasskube/trieve/configmap.yaml

@@ -16,6 +16,7 @@ data:
   SMTP_PASSWORD: "FROM SECRET"
   SMTP_EMAIL_ADDRESS: "[email protected]"
   OPENAI_API_KEY: "sk-************************************************"
+  ANTHROPIC_API_KEY: "sk-************************************************"
   LLM_API_KEY: "sk-************************************************"
   SECRET_KEY: "01234012340123401234012340123401234012340123401234012340123401234012340123401234" #generate
   SALT: "" # OPTIOANL
@@ -47,7 +48,8 @@ data:
   CLICKHOUSE_DB: "default"
   CLICKHOUSE_USER: "default"
   CLICKHOUSE_PASSWORD: "password"
+  CLICKHOUSE_DSN: "clickhouse://default:password@clickhouse-trieve-clickhouse:8123/default" # Used by Cronjobs
   JINA_CODE_API_KEY: ""
   RUST_LOG: "INFO"
   BM25_ACTIVE: "true"
-  CREATE_QDRANT_COLLECTIONS: "true"
+  CREATE_QDRANT_COLLECTIONS: "true"

glasskube/trieve/cronjobs/clickhouse-collapse.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: collapsing-queries-cronjob
+spec:
+  schedule: "*/30 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: collapsing-queries
+              image: trieve/clickhouse-collapse-query-script
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never
+

glasskube/trieve/cronjobs/cluster-queries.yaml

@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cluster-queries-cronjob
+spec:
+  schedule: "1 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: cluster-queries
+              image: trieve/clickhouse-clustering
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: ANTHROPIC_API_KEY
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: ANTHROPIC_API_KEY
+                - name: CLICKHOUSE_DSN
+                  valueFrom:
+                    configMapKeyRef:
+                      name: trieve-server-config
+                      key: CLICKHOUSE_DSN
+          restartPolicy: Never

glasskube/trieve/cronjobs/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - word-id-cronjob.yaml
+  - cluster-queries.yaml
+  - clickhouse-collapse.yaml

glasskube/trieve/cronjobs/word-id-cronjob.yaml

@@ -0,0 +1,31 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: trieve-word-id-cronjob
+spec:
+  schedule: "0 0 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: word-id-cronjob
+              image: trieve/word-id-cronjob
+              ports:
+                - containerPort: 8090
+              env:
+                - name: DATABASE_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-postgresql-server-app
+                      key: uri
+                - name: QDRANT_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: trieve-qdrant-qdrant-apikey
+                      key: api-key
+              envFrom:
+                - configMapRef:
+                    name: trieve-server-config

glasskube/trieve/kustomization.yaml

@@ -10,8 +10,8 @@ resources:
   - server
   - ui
   - worker
-  - ingest
   - embeddings
+  - cronjobs
 
 images:
   - name: ghcr.io/huggingface/text-embeddings-inference
@@ -39,3 +39,9 @@ images:
   - name: trieve/group-worker
     newName: trieve/group_worker
     newTag: sha-d8b18ca
+  - name: trieve/word-id-cronjob
+    newTag: sha-d8b18ca
+  - name: trieve/clickhouse-collapse-query-script
+    newTag: sha-5cdcd38
+  - name: trieve/clickhouse-clustering
+    newTag: sha-e55860a

glasskube/trieve/ui/configmap.yaml

@@ -8,3 +8,4 @@ data:
   VITE_DASHBOARD_URL: "http://dashboard.localtrieve.com"
   VITE_CHAT_UI_URL: "http://chat.localtrieve.com"
   VITE_ANALYTICS_UI_URL: "http://analytics.localtrieve.com"
+  VITE_BM25_ACTIVE: "true"

glasskube/trieve/worker/bktree-worker/kustomization.yaml

@@ -0,0 +1,8 @@
+commonLabels:
+  app.kubernetes.io/component: worker-bktree
+nameSuffix: -bktree
+resources:
+  - ../base
+images:
+  - name: worker
+    newName: trieve/bktree-worker

glasskube/trieve/worker/group/kustomization.yaml

@@ -5,4 +5,4 @@ resources:
   - ../base
 images:
   - name: worker
-    newName: trieve/file-worker
+    newName: trieve/group-worker

glasskube/trieve/worker/ingest/kustomization.yaml

@@ -0,0 +1,8 @@
+commonLabels:
+  app.kubernetes.io/component: worker-file
+nameSuffix: -ingest
+resources:
+  - ../base
+images:
+  - name: worker
+    newName: trieve/ingest

glasskube/trieve/worker/kustomization.yaml

@@ -1,5 +1,10 @@
 resources:
+  - bktree-worker
   - delete
   - file
   - group
+  - ingest
   - sync-qdrant
+  - reindexer
+  - queue-bm25
+  - word-worker

glasskube/trieve/worker/queue-bm25/kustomization.yaml

@@ -0,0 +1,16 @@
+commonLabels:
+  app.kubernetes.io/component: worker-queuebm25
+nameSuffix: -bm25
+resources:
+  - ../base
+images:
+  - name: worker
+    newName: trieve/queue-bm25-migration
+patches:
+  - target:
+      kind: Deployment
+      name: trieve-worker
+    patch: |-
+      - op: add
+        path: /spec/replicas
+        value: 0 # this deployment functions more like a "Job"

glasskube/trieve/worker/reindexer/kustomization.yaml

@@ -0,0 +1,8 @@
+commonLabels:
+  app.kubernetes.io/component: worker-reindexer
+nameSuffix: -reindexer
+resources:
+  - ../base
+images:
+  - name: worker
+    newName: trieve/reindex-worker

glasskube/trieve/worker/word-worker/kustomization.yaml

@@ -0,0 +1,8 @@
+commonLabels:
+  app.kubernetes.io/component: worker-words
+nameSuffix: -word
+resources:
+  - ../base
+images:
+  - name: worker
+    newName: trieve/word-worker