From adee0f64f68803a09fe98ce6f0c2e83f70b85702 Mon Sep 17 00:00:00 2001 From: cdxker Date: Fri, 30 Aug 2024 14:45:11 -0700 Subject: [PATCH 1/3] faet: added more workers, moved ingest to the workers template --- glasskube/README.md | 10 +++++-- glasskube/trieve/ingest/deployment.yaml | 29 ------------------- glasskube/trieve/ingest/kustomization.yaml | 4 --- glasskube/trieve/kustomization.yaml | 1 - .../worker/bktree-worker/kustomization.yaml | 8 +++++ .../trieve/worker/group/kustomization.yaml | 2 +- .../trieve/worker/ingest/kustomization.yaml | 8 +++++ glasskube/trieve/worker/kustomization.yaml | 5 ++++ .../worker/queue-bm25/kustomization.yaml | 16 ++++++++++ .../worker/reindexer/kustomization.yaml | 8 +++++ .../worker/word-worker/kustomization.yaml | 8 +++++ 11 files changed, 62 insertions(+), 37 deletions(-) delete mode 100644 glasskube/trieve/ingest/deployment.yaml delete mode 100644 glasskube/trieve/ingest/kustomization.yaml create mode 100644 glasskube/trieve/worker/bktree-worker/kustomization.yaml create mode 100644 glasskube/trieve/worker/ingest/kustomization.yaml create mode 100644 glasskube/trieve/worker/queue-bm25/kustomization.yaml create mode 100644 glasskube/trieve/worker/reindexer/kustomization.yaml create mode 100644 glasskube/trieve/worker/word-worker/kustomization.yaml diff --git a/glasskube/README.md b/glasskube/README.md index dbacd26add..66219476c0 100644 --- a/glasskube/README.md +++ b/glasskube/README.md @@ -70,9 +70,15 @@ Open [dashboard.localtrieve.com](http://dashboard.localtrieve.com) in your brows ## Optional commands -Scale down all embedding servers: +#### Scale down all embedding servers: ```shell kubectl scale deployment trieve-embedding-bgem3 trieve-embedding-jina trieve-embedding-reranker trieve-embedding-splade-doc trieve-embedding-splade-query --replicas 0 -``` \ No newline at end of file +``` + +#### Deleting the cluster + +```shell +minikube delete -p trieve +``` diff --git a/glasskube/trieve/ingest/deployment.yaml b/glasskube/trieve/ingest/deployment.yaml deleted file mode 100644 index 2d4dfcb89c..0000000000 --- a/glasskube/trieve/ingest/deployment.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trieve-ingest -spec: - selector: {} - template: - spec: - containers: - - name: ingest - image: trieve/ingest - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: trieve-postgresql-server-app - key: uri - - name: QDRANT_API_KEY - valueFrom: - secretKeyRef: - name: trieve-qdrant-qdrant-apikey - key: api-key - envFrom: - - configMapRef: - name: trieve-server-config - resources: - limits: - cpu: "1" - memory: 100Mi diff --git a/glasskube/trieve/ingest/kustomization.yaml b/glasskube/trieve/ingest/kustomization.yaml deleted file mode 100644 index e06a289ae1..0000000000 --- a/glasskube/trieve/ingest/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -commonLabels: - app.kubernetes.io/component: ingest -resources: - - deployment.yaml \ No newline at end of file diff --git a/glasskube/trieve/kustomization.yaml b/glasskube/trieve/kustomization.yaml index 50a09d4f55..c1264b0d22 100644 --- a/glasskube/trieve/kustomization.yaml +++ b/glasskube/trieve/kustomization.yaml @@ -10,7 +10,6 @@ resources: - server - ui - worker - - ingest - embeddings images: diff --git a/glasskube/trieve/worker/bktree-worker/kustomization.yaml b/glasskube/trieve/worker/bktree-worker/kustomization.yaml new file mode 100644 index 0000000000..4e46a0558d --- /dev/null +++ b/glasskube/trieve/worker/bktree-worker/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-bktree +nameSuffix: -bktree +resources: + - ../base +images: + - name: worker + newName: trieve/bktree-worker diff --git a/glasskube/trieve/worker/group/kustomization.yaml b/glasskube/trieve/worker/group/kustomization.yaml index a72a049d5e..731c88e82b 100644 --- a/glasskube/trieve/worker/group/kustomization.yaml +++ b/glasskube/trieve/worker/group/kustomization.yaml @@ -5,4 +5,4 @@ resources: - ../base images: - name: worker - newName: trieve/file-worker + newName: trieve/group-worker diff --git a/glasskube/trieve/worker/ingest/kustomization.yaml b/glasskube/trieve/worker/ingest/kustomization.yaml new file mode 100644 index 0000000000..3bb2a2014a --- /dev/null +++ b/glasskube/trieve/worker/ingest/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-file +nameSuffix: -ingest +resources: + - ../base +images: + - name: worker + newName: trieve/ingest diff --git a/glasskube/trieve/worker/kustomization.yaml b/glasskube/trieve/worker/kustomization.yaml index 86b43a1fa3..9cf542d757 100644 --- a/glasskube/trieve/worker/kustomization.yaml +++ b/glasskube/trieve/worker/kustomization.yaml @@ -1,5 +1,10 @@ resources: + - bktree-worker - delete - file - group + - ingest - sync-qdrant + - reindexer + - queue-bm25 + - word-worker diff --git a/glasskube/trieve/worker/queue-bm25/kustomization.yaml b/glasskube/trieve/worker/queue-bm25/kustomization.yaml new file mode 100644 index 0000000000..a7d58cf5f4 --- /dev/null +++ b/glasskube/trieve/worker/queue-bm25/kustomization.yaml @@ -0,0 +1,16 @@ +commonLabels: + app.kubernetes.io/component: worker-queuebm25 +nameSuffix: -bm25 +resources: + - ../base +images: + - name: worker + newName: trieve/queue-bm25-migration +patches: + - target: + kind: Deployment + name: trieve-worker + patch: |- + - op: add + path: /spec/replicas + value: 0 # this deployment functions more like a "Job" diff --git a/glasskube/trieve/worker/reindexer/kustomization.yaml b/glasskube/trieve/worker/reindexer/kustomization.yaml new file mode 100644 index 0000000000..fb81147f13 --- /dev/null +++ b/glasskube/trieve/worker/reindexer/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-reindexer +nameSuffix: -reindexer +resources: + - ../base +images: + - name: worker + newName: trieve/reindex-worker diff --git a/glasskube/trieve/worker/word-worker/kustomization.yaml b/glasskube/trieve/worker/word-worker/kustomization.yaml new file mode 100644 index 0000000000..1431c5189a --- /dev/null +++ b/glasskube/trieve/worker/word-worker/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-words +nameSuffix: -word +resources: + - ../base +images: + - name: worker + newName: trieve/word-worker From 1d94ede0bcc0c612de72537d2ee09ed7b7bcf681 Mon Sep 17 00:00:00 2001 From: cdxker Date: Fri, 30 Aug 2024 15:49:16 -0700 Subject: [PATCH 2/3] feat: added cronjobs into glasskube repo --- glasskube/trieve/configmap.yaml | 4 ++- .../trieve/cronjobs/clickhouse-collapse.yaml | 23 ++++++++++++++ .../trieve/cronjobs/cluster-queries.yaml | 27 ++++++++++++++++ glasskube/trieve/cronjobs/kustomization.yaml | 4 +++ .../trieve/cronjobs/word-id-cronjob.yaml | 31 +++++++++++++++++++ glasskube/trieve/kustomization.yaml | 7 +++++ glasskube/trieve/ui/configmap.yaml | 1 + 7 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 glasskube/trieve/cronjobs/clickhouse-collapse.yaml create mode 100644 glasskube/trieve/cronjobs/cluster-queries.yaml create mode 100644 glasskube/trieve/cronjobs/kustomization.yaml create mode 100644 glasskube/trieve/cronjobs/word-id-cronjob.yaml diff --git a/glasskube/trieve/configmap.yaml b/glasskube/trieve/configmap.yaml index 24c96caaba..25e10c55a4 100644 --- a/glasskube/trieve/configmap.yaml +++ b/glasskube/trieve/configmap.yaml @@ -16,6 +16,7 @@ data: SMTP_PASSWORD: "FROM SECRET" SMTP_EMAIL_ADDRESS: "noreply@trieve.ai" OPENAI_API_KEY: "sk-************************************************" + ANTHROPIC_API_KEY: "sk-************************************************" LLM_API_KEY: "sk-************************************************" SECRET_KEY: "01234012340123401234012340123401234012340123401234012340123401234012340123401234" #generate SALT: "" # OPTIOANL @@ -47,7 +48,8 @@ data: CLICKHOUSE_DB: "default" CLICKHOUSE_USER: "default" CLICKHOUSE_PASSWORD: "password" + CLICKHOUSE_DSN: "clickhouse://default:password@clickhouse-trieve-clickhouse:8123/default" # Used by Cronjobs JINA_CODE_API_KEY: "" RUST_LOG: "INFO" BM25_ACTIVE: "true" - CREATE_QDRANT_COLLECTIONS: "true" \ No newline at end of file + CREATE_QDRANT_COLLECTIONS: "true" diff --git a/glasskube/trieve/cronjobs/clickhouse-collapse.yaml b/glasskube/trieve/cronjobs/clickhouse-collapse.yaml new file mode 100644 index 0000000000..453e7b17ba --- /dev/null +++ b/glasskube/trieve/cronjobs/clickhouse-collapse.yaml @@ -0,0 +1,23 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: collapsing-queries-cronjob +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: collapsing-queries + image: trieve/clickhouse-collapse-query-script + imagePullPolicy: IfNotPresent + env: + - name: CLICKHOUSE_DSN + valueFrom: + configMapKeyRef: + name: trieve-server-config + key: CLICKHOUSE_DSN + restartPolicy: Never + diff --git a/glasskube/trieve/cronjobs/cluster-queries.yaml b/glasskube/trieve/cronjobs/cluster-queries.yaml new file mode 100644 index 0000000000..440edcd273 --- /dev/null +++ b/glasskube/trieve/cronjobs/cluster-queries.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-queries-cronjob +spec: + schedule: "1 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: cluster-queries + image: trieve/clickhouse-clustering + imagePullPolicy: IfNotPresent + env: + - name: ANTHROPIC_API_KEY + valueFrom: + configMapKeyRef: + name: trieve-server-config + key: ANTHROPIC_API_KEY + - name: CLICKHOUSE_DSN + valueFrom: + configMapKeyRef: + name: trieve-server-config + key: CLICKHOUSE_DSN + restartPolicy: Never diff --git a/glasskube/trieve/cronjobs/kustomization.yaml b/glasskube/trieve/cronjobs/kustomization.yaml new file mode 100644 index 0000000000..a1c5c86f44 --- /dev/null +++ b/glasskube/trieve/cronjobs/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - word-id-cronjob.yaml + - cluster-queries.yaml + - clickhouse-collapse.yaml diff --git a/glasskube/trieve/cronjobs/word-id-cronjob.yaml b/glasskube/trieve/cronjobs/word-id-cronjob.yaml new file mode 100644 index 0000000000..eaa070d493 --- /dev/null +++ b/glasskube/trieve/cronjobs/word-id-cronjob.yaml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: trieve-word-id-cronjob +spec: + schedule: "0 0 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: word-id-cronjob + image: trieve/word-id-cronjob + ports: + - containerPort: 8090 + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: trieve-postgresql-server-app + key: uri + - name: QDRANT_API_KEY + valueFrom: + secretKeyRef: + name: trieve-qdrant-qdrant-apikey + key: api-key + envFrom: + - configMapRef: + name: trieve-server-config diff --git a/glasskube/trieve/kustomization.yaml b/glasskube/trieve/kustomization.yaml index c1264b0d22..0f605ac47b 100644 --- a/glasskube/trieve/kustomization.yaml +++ b/glasskube/trieve/kustomization.yaml @@ -11,6 +11,7 @@ resources: - ui - worker - embeddings + - cronjobs images: - name: ghcr.io/huggingface/text-embeddings-inference @@ -38,3 +39,9 @@ images: - name: trieve/group-worker newName: trieve/group_worker newTag: sha-d8b18ca + - name: trieve/word-id-cronjob + newTag: sha-d8b18ca + - name: trieve/clickhouse-collapse-query-script + newTag: sha-5cdcd38 + - name: trieve/clickhouse-clustering + newTag: sha-e55860a diff --git a/glasskube/trieve/ui/configmap.yaml b/glasskube/trieve/ui/configmap.yaml index ef870083b1..eddf94a004 100644 --- a/glasskube/trieve/ui/configmap.yaml +++ b/glasskube/trieve/ui/configmap.yaml @@ -8,3 +8,4 @@ data: VITE_DASHBOARD_URL: "http://dashboard.localtrieve.com" VITE_CHAT_UI_URL: "http://chat.localtrieve.com" VITE_ANALYTICS_UI_URL: "http://analytics.localtrieve.com" + VITE_BM25_ACTIVE: "true" From ef3a81abc9c3a1c90b53034dd9e516d7485a1230 Mon Sep 17 00:00:00 2001 From: cdxker Date: Fri, 30 Aug 2024 14:49:15 -0700 Subject: [PATCH 3/3] ops: make docs CI action only run on client updates --- .github/workflows/ts-sdk-gh-pages.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ts-sdk-gh-pages.yml b/.github/workflows/ts-sdk-gh-pages.yml index 0081f15bc6..4d0fb46bea 100644 --- a/.github/workflows/ts-sdk-gh-pages.yml +++ b/.github/workflows/ts-sdk-gh-pages.yml @@ -2,6 +2,12 @@ name: Deploy TS-SDK docs to github pages on: push: + paths: + - 'clients/ts-sdk/**' + branches: ["main"] + pull_request: + paths: + - 'clients/ts-sdk/**' branches: ["main"] jobs: