[nl server] get default and enabled indexes from deployment (#4243)

Get default index and enabled indexes from deployment because this could be instance specific
datacommonsorg · May 15, 2024 · 3a2854f · 3a2854f
1 parent f7433df
commit 3a2854f
Show file tree

Hide file tree

Showing 21 changed files with 384 additions and 223 deletions.
diff --git a/deploy/helm_charts/dc_website/templates/config_maps.yaml b/deploy/helm_charts/dc_website/templates/config_maps.yaml
@@ -81,7 +81,7 @@ metadata:
 data:
   embeddings.yaml: {{ required "NL embeddings file is required" .Values.nl.embeddings | quote }}
   models.yaml: {{ required "NL models file is required" .Values.nl.models | quote }}
-  vertex_ai_models.json: {{ .Values.nl.vertex_ai_models | toJson | quote }}
+  embeddings_spec.json: {{ .Values.nl.embeddingsSpec | toJson | quote }}
 {{- end }}
 
 {{- if .Values.website.redis.enabled }}

diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml
@@ -136,7 +136,11 @@ nl:
   models:
   memory: "2G"
   workers: 1
-  vertex_ai_models:
+  embeddingsSpec:
+    defaultIndex: ""
+    enabledIndexes: []
+    vertexAIModels:
+    enableReranking: false
 
 
 ###############################################################################

diff --git a/deploy/helm_charts/envs/autopush.yaml b/deploy/helm_charts/envs/autopush.yaml
@@ -44,27 +44,40 @@ serviceAccount:
 
 nl:
   enabled: true
-  vertex_ai_models:
-    dc-all-minilm-l6-v2-model:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "8518340991868993536"
-    uae-large-v1-model:
-      project_id: datcom-nl
-      location: us-central1
-      prediction_endpoint_id: "8110162693219942400"
-    sfr-embedding-mistral-model:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "224012300019826688"
-    cross-encoder-ms-marco-miniilm-l6-v2:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "3977846152316846080"
-    cross-encoder-mxbai-rerank-base-v1:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "284894457873039360"
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "base_uae_mem",
+      "bio_ft",
+      "medium_ft",
+      "medium_lance_ft",
+      "medium_vertex_ft",
+      "medium_vertex_mistral",
+      "sdg_ft",
+      "undata_ft",
+    ]
+    vertexAIModels:
+      dc-all-minilm-l6-v2-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "8518340991868993536"
+      uae-large-v1-model:
+        project_id: datcom-nl
+        location: us-central1
+        prediction_endpoint_id: "8110162693219942400"
+      sfr-embedding-mistral-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "224012300019826688"
+      cross-encoder-ms-marco-miniilm-l6-v2:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "3977846152316846080"
+      cross-encoder-mxbai-rerank-base-v1:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "284894457873039360"
+    enableReranking: true
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/bard.yaml b/deploy/helm_charts/envs/bard.yaml
@@ -44,6 +44,14 @@ nl:
   enabled: true
   memory: "10G"
   workers: 3
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 nodejs:
   enabled: true

diff --git a/deploy/helm_charts/envs/biomedical.yaml b/deploy/helm_charts/envs/biomedical.yaml
@@ -42,3 +42,11 @@ serviceGroups:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "bio_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
diff --git a/deploy/helm_charts/envs/climate_trace.yaml b/deploy/helm_charts/envs/climate_trace.yaml
@@ -34,6 +34,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/dev.yaml b/deploy/helm_charts/envs/dev.yaml
@@ -41,27 +41,40 @@ serviceGroups:
 
 nl:
   enabled: true
-  vertex_ai_models:
-    dc-all-minilm-l6-v2-model:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "8518340991868993536"
-    uae-large-v1-model:
-      project_id: datcom-nl
-      location: us-central1
-      prediction_endpoint_id: "1400502935879680000"
-    sfr-embedding-mistral-model:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "224012300019826688"
-    cross-encoder-ms-marco-miniilm-l6-v2:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "3977846152316846080"
-    cross-encoder-mxbai-rerank-base-v1:
-      project_id: datcom-website-dev
-      location: us-central1
-      prediction_endpoint_id: "284894457873039360"
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "base_uae_mem",
+      "bio_ft",
+      "medium_ft",
+      "medium_lance_ft",
+      "medium_vertex_ft",
+      "medium_vertex_mistral",
+      "sdg_ft",
+      "undata_ft",
+    ]
+    vertexAIModels:
+      dc-all-minilm-l6-v2-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "8518340991868993536"
+      uae-large-v1-model:
+        project_id: datcom-nl
+        location: us-central1
+        prediction_endpoint_id: "8110162693219942400"
+      sfr-embedding-mistral-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "224012300019826688"
+      cross-encoder-ms-marco-miniilm-l6-v2:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "3977846152316846080"
+      cross-encoder-mxbai-rerank-base-v1:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "284894457873039360"
+    enableReranking: true
 
 nodejs:
   enabled: true

diff --git a/deploy/helm_charts/envs/internal.yaml b/deploy/helm_charts/envs/internal.yaml
@@ -36,6 +36,14 @@ ingress:
   certName: website-ssl-certificate
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/magic_eye.yaml b/deploy/helm_charts/envs/magic_eye.yaml
@@ -37,6 +37,14 @@ ingress:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/prod.yaml b/deploy/helm_charts/envs/prod.yaml
@@ -41,6 +41,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml
@@ -32,6 +32,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/unsdg.yaml b/deploy/helm_charts/envs/unsdg.yaml
@@ -40,6 +40,14 @@ ingress:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "sdg_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/unsdg_staging.yaml b/deploy/helm_charts/envs/unsdg_staging.yaml
@@ -37,6 +37,14 @@ mixer:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "sdg_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/worldbank.yaml b/deploy/helm_charts/envs/worldbank.yaml
@@ -49,6 +49,14 @@ serviceGroups:
       memoryLimit: "8G"
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 svg:
   blocklistFile: ["dc/g/Uncategorized", "oecd/g/OECD"]