From 2e2e0e982acc47380238182cacd1118f0e40eda9 Mon Sep 17 00:00:00 2001 From: Philip Miglinci Date: Fri, 30 Aug 2024 12:10:58 -0700 Subject: [PATCH] feat: refactor worker to base deployment Signed-off-by: Philip Miglinci --- glasskube/trieve/configmap.yaml | 2 -- .../trieve/embeddings/base/kustomization.yaml | 2 ++ glasskube/trieve/kustomization.yaml | 5 +-- glasskube/trieve/sync-qdrant/deployment.yaml | 31 ------------------- .../trieve/sync-qdrant/kustomization.yaml | 7 ----- .../{delete-worker => base}/deployment.yaml | 10 +++--- .../trieve/worker/base/kustomization.yaml | 2 ++ .../worker/delete-worker/kustomization.yaml | 4 --- .../trieve/worker/delete/kustomization.yaml | 8 +++++ .../trieve/worker/file-worker/deployment.yaml | 29 ----------------- .../worker/file-worker/kustomization.yaml | 4 --- .../trieve/worker/file/kustomization.yaml | 8 +++++ .../worker/group-worker/deployment.yaml | 29 ----------------- .../worker/group-worker/kustomization.yaml | 4 --- .../trieve/worker/group/kustomization.yaml | 8 +++++ glasskube/trieve/worker/kustomization.yaml | 7 +++-- .../worker/sync-qdrant/kustomization.yaml | 16 ++++++++++ 17 files changed, 57 insertions(+), 119 deletions(-) delete mode 100644 glasskube/trieve/sync-qdrant/deployment.yaml delete mode 100644 glasskube/trieve/sync-qdrant/kustomization.yaml rename glasskube/trieve/worker/{delete-worker => base}/deployment.yaml (80%) create mode 100644 glasskube/trieve/worker/base/kustomization.yaml delete mode 100644 glasskube/trieve/worker/delete-worker/kustomization.yaml create mode 100644 glasskube/trieve/worker/delete/kustomization.yaml delete mode 100644 glasskube/trieve/worker/file-worker/deployment.yaml delete mode 100644 glasskube/trieve/worker/file-worker/kustomization.yaml create mode 100644 glasskube/trieve/worker/file/kustomization.yaml delete mode 100644 glasskube/trieve/worker/group-worker/deployment.yaml delete mode 100644 glasskube/trieve/worker/group-worker/kustomization.yaml create mode 100644 glasskube/trieve/worker/group/kustomization.yaml create mode 100644 glasskube/trieve/worker/sync-qdrant/kustomization.yaml diff --git a/glasskube/trieve/configmap.yaml b/glasskube/trieve/configmap.yaml index b0bed708da..24c96caaba 100644 --- a/glasskube/trieve/configmap.yaml +++ b/glasskube/trieve/configmap.yaml @@ -8,11 +8,9 @@ data: BASE_SERVER_URL: "http://api.localtrieve.com" REDIS_URL: "redis://redis:6379" QDRANT_URL: "http://trieve-qdrant-qdrant:6334" - # QDRANT_API_KEY: "FROM SECRET" # FROM SECRET QUANTIZE_VECTORS: "false" REPLICATION_FACTOR: "2" VECTOR_SIZES: "384,512,768,1024,1536,3072" - # DATABASE_URL: "FROM_SCERET" SMTP_RELAY: "smtp.fastmail.com" SMTP_USERNAME: "noreply@trieve.ai" SMTP_PASSWORD: "FROM SECRET" diff --git a/glasskube/trieve/embeddings/base/kustomization.yaml b/glasskube/trieve/embeddings/base/kustomization.yaml index 6d1374a18e..706be98a60 100644 --- a/glasskube/trieve/embeddings/base/kustomization.yaml +++ b/glasskube/trieve/embeddings/base/kustomization.yaml @@ -1,3 +1,5 @@ resources: - deployment.yaml - service.yaml + + diff --git a/glasskube/trieve/kustomization.yaml b/glasskube/trieve/kustomization.yaml index ad13fc100c..50a09d4f55 100644 --- a/glasskube/trieve/kustomization.yaml +++ b/glasskube/trieve/kustomization.yaml @@ -2,7 +2,7 @@ namespace: default commonLabels: app.kubernetes.io/name: trieve app.kubernetes.io/part-of: trieve - app.kubernetes.io/instance: test + app.kubernetes.io/instance: trieve resources: - configmap.yaml - clickhouse.yaml @@ -10,7 +10,6 @@ resources: - server - ui - worker - - sync-qdrant # really needed? - ingest - embeddings @@ -19,6 +18,8 @@ images: newTag: cpu-1.4 - name: trieve/ingest newTag: sha-d8b18ca + - name: trieve/sync_qdrant + newTag: sha-d8b18ca - name: trieve/server newTag: sha-d8b18ca - name: trieve/analytics-site diff --git a/glasskube/trieve/sync-qdrant/deployment.yaml b/glasskube/trieve/sync-qdrant/deployment.yaml deleted file mode 100644 index ee7a74161a..0000000000 --- a/glasskube/trieve/sync-qdrant/deployment.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trieve-sync-qdrant -spec: - # TODO: Why is repliacs set to "0" in the helm chart? is this even needed? - replicas: 0 # Set to 0 by default - selector: {} - template: - spec: - containers: - - name: sync-qdrant - image: trieve/sync_qdrant - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: trieve-postgresql-server-app - key: uri - - name: QDRANT_API_KEY - valueFrom: - secretKeyRef: - name: trieve-qdrant-qdrant-apikey - key: api-key - envFrom: - - configMapRef: - name: trieve-server-config - resources: - limits: - cpu: "1" - memory: 100Mi diff --git a/glasskube/trieve/sync-qdrant/kustomization.yaml b/glasskube/trieve/sync-qdrant/kustomization.yaml deleted file mode 100644 index 6076c5aa5a..0000000000 --- a/glasskube/trieve/sync-qdrant/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -commonLabels: - app.kubernetes.io/component: sync-qdrant -resources: - - deployment.yaml -images: - - name: trieve/sync_qdrant - newTag: latest # TODO: change to pinned version diff --git a/glasskube/trieve/worker/delete-worker/deployment.yaml b/glasskube/trieve/worker/base/deployment.yaml similarity index 80% rename from glasskube/trieve/worker/delete-worker/deployment.yaml rename to glasskube/trieve/worker/base/deployment.yaml index 17bdef0a02..87571e4bb3 100644 --- a/glasskube/trieve/worker/delete-worker/deployment.yaml +++ b/glasskube/trieve/worker/base/deployment.yaml @@ -1,14 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: trieve-delete-worker + name: trieve-worker spec: selector: {} template: spec: containers: - - name: delete-worker - image: trieve/delete-worker + - name: worker + image: worker + ports: + - containerPort: 80 env: - name: DATABASE_URL valueFrom: @@ -26,4 +28,4 @@ spec: resources: limits: cpu: "1" - memory: 100Mi + memory: 200Mi diff --git a/glasskube/trieve/worker/base/kustomization.yaml b/glasskube/trieve/worker/base/kustomization.yaml new file mode 100644 index 0000000000..9519a26d6d --- /dev/null +++ b/glasskube/trieve/worker/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - deployment.yaml diff --git a/glasskube/trieve/worker/delete-worker/kustomization.yaml b/glasskube/trieve/worker/delete-worker/kustomization.yaml deleted file mode 100644 index 646d8ff94c..0000000000 --- a/glasskube/trieve/worker/delete-worker/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -commonLabels: - app.kubernetes.io/component: delete-worker -resources: - - deployment.yaml \ No newline at end of file diff --git a/glasskube/trieve/worker/delete/kustomization.yaml b/glasskube/trieve/worker/delete/kustomization.yaml new file mode 100644 index 0000000000..64ee8cb0d7 --- /dev/null +++ b/glasskube/trieve/worker/delete/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-delete +nameSuffix: -delete +resources: + - ../base +images: + - name: worker + newName: trieve/delete-worker diff --git a/glasskube/trieve/worker/file-worker/deployment.yaml b/glasskube/trieve/worker/file-worker/deployment.yaml deleted file mode 100644 index 03f63e43ba..0000000000 --- a/glasskube/trieve/worker/file-worker/deployment.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trieve-file-worker -spec: - selector: {} - template: - spec: - containers: - - name: file-worker - image: trieve/file-worker - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: trieve-postgresql-server-app - key: uri - - name: QDRANT_API_KEY - valueFrom: - secretKeyRef: - name: trieve-qdrant-qdrant-apikey - key: api-key - envFrom: - - configMapRef: - name: trieve-server-config - resources: - limits: - cpu: "1" - memory: 100Mi diff --git a/glasskube/trieve/worker/file-worker/kustomization.yaml b/glasskube/trieve/worker/file-worker/kustomization.yaml deleted file mode 100644 index 80085df0ee..0000000000 --- a/glasskube/trieve/worker/file-worker/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -commonLabels: - app.kubernetes.io/component: file-worker -resources: - - deployment.yaml \ No newline at end of file diff --git a/glasskube/trieve/worker/file/kustomization.yaml b/glasskube/trieve/worker/file/kustomization.yaml new file mode 100644 index 0000000000..d0b3479c94 --- /dev/null +++ b/glasskube/trieve/worker/file/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-file +nameSuffix: -file +resources: + - ../base +images: + - name: worker + newName: trieve/file-worker diff --git a/glasskube/trieve/worker/group-worker/deployment.yaml b/glasskube/trieve/worker/group-worker/deployment.yaml deleted file mode 100644 index 3e4d185389..0000000000 --- a/glasskube/trieve/worker/group-worker/deployment.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trieve-group-worker -spec: - selector: {} - template: - spec: - containers: - - name: group-worker - image: trieve/group-worker - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: trieve-postgresql-server-app - key: uri - - name: QDRANT_API_KEY - valueFrom: - secretKeyRef: - name: trieve-qdrant-qdrant-apikey - key: api-key - envFrom: - - configMapRef: - name: trieve-server-config - resources: - limits: - cpu: "1" - memory: 100Mi diff --git a/glasskube/trieve/worker/group-worker/kustomization.yaml b/glasskube/trieve/worker/group-worker/kustomization.yaml deleted file mode 100644 index d242287ec7..0000000000 --- a/glasskube/trieve/worker/group-worker/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -commonLabels: - app.kubernetes.io/component: group-worker -resources: - - deployment.yaml \ No newline at end of file diff --git a/glasskube/trieve/worker/group/kustomization.yaml b/glasskube/trieve/worker/group/kustomization.yaml new file mode 100644 index 0000000000..a72a049d5e --- /dev/null +++ b/glasskube/trieve/worker/group/kustomization.yaml @@ -0,0 +1,8 @@ +commonLabels: + app.kubernetes.io/component: worker-file +nameSuffix: -group +resources: + - ../base +images: + - name: worker + newName: trieve/file-worker diff --git a/glasskube/trieve/worker/kustomization.yaml b/glasskube/trieve/worker/kustomization.yaml index 5a7e8c26aa..86b43a1fa3 100644 --- a/glasskube/trieve/worker/kustomization.yaml +++ b/glasskube/trieve/worker/kustomization.yaml @@ -1,4 +1,5 @@ resources: - - delete-worker - - file-worker - - group-worker + - delete + - file + - group + - sync-qdrant diff --git a/glasskube/trieve/worker/sync-qdrant/kustomization.yaml b/glasskube/trieve/worker/sync-qdrant/kustomization.yaml new file mode 100644 index 0000000000..67e27b023b --- /dev/null +++ b/glasskube/trieve/worker/sync-qdrant/kustomization.yaml @@ -0,0 +1,16 @@ +commonLabels: + app.kubernetes.io/component: worker-sync-qdrant +nameSuffix: -sync-qdrant +resources: + - ../base +images: + - name: worker + newName: trieve/sync_qdrant +patches: + - target: + kind: Deployment + name: trieve-worker + patch: |- + - op: add + path: /spec/replicas + value: 0 # this deployment functions more like a "Job" \ No newline at end of file