Merge branch 'cache-geojson' of github.com:dwnoble/website into cache…

…-geojson
datacommonsorg · May 16, 2024 · 81ad59f · 81ad59f
2 parents 713bd49 + 0cda4de
commit 81ad59f
Show file tree

Hide file tree

Showing 135 changed files with 206,412 additions and 29,631 deletions.
diff --git a/custom_dc/cloudsql_env.list b/custom_dc/cloudsql_env.list
@@ -1,12 +1,27 @@
-FLASK_ENV=custom
-USE_CLOUDSQL=true
-ENABLE_MODEL=true
-GOOGLE_CLOUD_PROJECT=
+
+# API key for accessing the base Data Commons API.
 DC_API_KEY=
+# API key for accessing Maps and Places APIs for exploration tools.
 MAPS_API_KEY=
-GCS_DATA_PATH=
+# Use Google Cloud SQL as the database. 
+USE_CLOUDSQL=true
+# The project ID of the Google Cloud Platform project for the custom Data Commons.
+GOOGLE_CLOUD_PROJECT=
+# The name of the Cloud SQL instance, in the form <PROJECT_ID>:<LOCATION>:<INSTANCE_ID>.     
 CLOUDSQL_INSTANCE=
+# The name of the Google Cloud SQL database.
+DB_NAME=datacommons
+# The name of a database user configured to access the Cloud SQL instance. It may be root or another user you have configured.
 DB_USER=
+# The password of the user specified in DB_USER.
 DB_PASS=
-ADMIN_SECRET=
+# (Optional) If used, the Redis Memorystore instance IP address.
 REDIS_HOST=
+# The data path of the files stored in Google Cloud Storage, in the form gs://<BUCKET>/<FOLDER>/.
+GCS_DATA_PATH=
+# (Optional) Secret token to authorize users to perform /admin page operations.
+ADMIN_SECRET=
+# Enable embeddings generation and natural language querying. 
+ENABLE_MODEL=true
+# The name of the parent directory for custom CSS, JS, HTML and image files.
+FLASK_ENV=custom
diff --git a/custom_dc/sqlite_env.list b/custom_dc/sqlite_env.list
@@ -1,6 +1,12 @@
-FLASK_ENV=custom
-USE_SQLITE=true
-ENABLE_MODEL=true
+# API key for accessing the base Data Commons API.
 DC_API_KEY=
+# API key for accessing Maps and Places APIs for exploration tools.
 MAPS_API_KEY=
-ADMIN_SECRET=
+# Use local SQLite as the database.
+USE_SQLITE=true
+# (Optional) Secret token to authorize users to perform /admin page operations.
+ADMIN_SECRET=
+# Enable embeddings generation and natural language querying. 
+ENABLE_MODEL=true
+# The name of the parent directory for custom CSS, JS, HTML and image files.
+FLASK_ENV=custom
diff --git a/deploy/helm_charts/dc_website/templates/config_maps.yaml b/deploy/helm_charts/dc_website/templates/config_maps.yaml
@@ -81,9 +81,9 @@ metadata:
 data:
   embeddings.yaml: {{ required "NL embeddings file is required" .Values.nl.embeddings | quote }}
   models.yaml: {{ required "NL models file is required" .Values.nl.models | quote }}
+  embeddings_spec.json: {{ .Values.nl.embeddingsSpec | toJson | quote }}
 {{- end }}
 
-
 {{- if .Values.website.redis.enabled }}
 ---
 kind: ConfigMap

diff --git a/deploy/helm_charts/dc_website/templates/single_cluster.yaml b/deploy/helm_charts/dc_website/templates/single_cluster.yaml
@@ -84,6 +84,42 @@ spec:
 
 {{- end }}
 
+{{- if .Values.ingress.enableMixer }}
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: website-mixer-service
+  namespace: {{ .Values.namespace.name }}
+  annotations:
+    # This is to get longer timeout for the Cloud Load Balancer.
+    cloud.google.com/backend-config: '{"ports": {"80":"website-mixer-backendconfig"}}'
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      targetPort: 8081
+      protocol: TCP
+      name: http
+  selector:
+    service: dc-mixer-default
+
+---
+
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  name: website-mixer-backendconfig
+  namespace: {{ .Values.namespace.name }}
+spec:
+  # Timeout for load balancer and the GKE pod connection
+  timeoutSec: 300
+  connectionDraining:
+    drainingTimeoutSec: 300
+
+{{- end }}
+
 ---
 
 # Note:
@@ -125,6 +161,24 @@ spec:
               port:
                 number: 8080
     {{- end }}
+    {{- if .Values.ingress.enableMixer }}
+    - http:
+        paths:
+        - path: /v2/*
+          pathType: ImplementationSpecific
+          backend:
+            service:
+              name: website-mixer-service
+              port:
+                number: 80
+        - path: /v1/*
+          pathType: ImplementationSpecific
+          backend:
+            service:
+              name: website-mixer-service
+              port:
+                number: 80
+    {{- end }}
 
 
 {{- end }}
diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml
@@ -136,6 +136,12 @@ nl:
   models:
   memory: "2G"
   workers: 1
+  embeddingsSpec:
+    defaultIndex: ""
+    enabledIndexes: []
+    vertexAIModels:
+    enableReranking: false
+
 
 ###############################################################################
 # Config for Stat Var Groups which is shared between Website and Mixer
@@ -157,7 +163,7 @@ nodejs:
   nodePool:
 
 ###############################################################################
-# Config for Periodic Testing CronJob
+# Config for Cron Testing Job
 ###############################################################################
 cronTesting:
   enabled: false
@@ -174,3 +180,5 @@ cronTesting:
   enableAdversarial: true
   # node pool to use
   nodePool:
+  # schedule to run the cron tests on
+  schedule:
diff --git a/deploy/helm_charts/envs/autopush.yaml b/deploy/helm_charts/envs/autopush.yaml
@@ -44,6 +44,42 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes:
+      [
+        "base_uae_mem",
+        "bio_ft",
+        "medium_ft",
+        "medium_lance_ft",
+        "medium_vertex_ft",
+        "medium_vertex_mistral",
+        "sdg_ft",
+        "undata_ft",
+        "undata_ilo_ft",
+      ]
+    vertexAIModels:
+      dc-all-minilm-l6-v2-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "8518340991868993536"
+      uae-large-v1-model:
+        project_id: datcom-nl
+        location: us-central1
+        prediction_endpoint_id: "8110162693219942400"
+      sfr-embedding-mistral-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "224012300019826688"
+      cross-encoder-ms-marco-miniilm-l6-v2:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "3977846152316846080"
+      cross-encoder-mxbai-rerank-base-v1:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "284894457873039360"
+    enableReranking: true
 
 serviceGroups:
   recon: null
@@ -64,3 +100,4 @@ cronTesting:
   enabled: true
   screenshotDomain: "autopush.datacommons.org"
   nodePool: "pool-1"
+  schedule: "0 */4 * * *"
diff --git a/deploy/helm_charts/envs/bard.yaml b/deploy/helm_charts/envs/bard.yaml
@@ -44,6 +44,14 @@ nl:
   enabled: true
   memory: "10G"
   workers: 3
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 nodejs:
   enabled: true
@@ -73,3 +81,4 @@ cronTesting:
   enabled: true
   enableSanity: false
   enableAdversarial: false
+  schedule: "0 */4 * * *"
diff --git a/deploy/helm_charts/envs/biomedical.yaml b/deploy/helm_charts/envs/biomedical.yaml
@@ -42,3 +42,11 @@ serviceGroups:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "bio_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
diff --git a/deploy/helm_charts/envs/climate_trace.yaml b/deploy/helm_charts/envs/climate_trace.yaml
@@ -34,6 +34,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/dev.yaml b/deploy/helm_charts/envs/dev.yaml
@@ -41,7 +41,47 @@ serviceGroups:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "base_uae_mem",
+      "bio_ft",
+      "medium_ft",
+      "medium_lance_ft",
+      "medium_vertex_ft",
+      "medium_vertex_mistral",
+      "sdg_ft",
+      "undata_ft",
+    ]
+    vertexAIModels:
+      dc-all-minilm-l6-v2-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "8518340991868993536"
+      uae-large-v1-model:
+        project_id: datcom-nl
+        location: us-central1
+        prediction_endpoint_id: "8110162693219942400"
+      sfr-embedding-mistral-model:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "224012300019826688"
+      cross-encoder-ms-marco-miniilm-l6-v2:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "3977846152316846080"
+      cross-encoder-mxbai-rerank-base-v1:
+        project_id: datcom-website-dev
+        location: us-central1
+        prediction_endpoint_id: "284894457873039360"
+    enableReranking: true
 
 nodejs:
   enabled: true
   replicas: 5
+
+cronTesting:
+  enabled: true
+  screenshotDomain: "dev.datacommons.org"
+  nodePool: "default-pool"
+  schedule: "* * 31 2 *"
diff --git a/deploy/helm_charts/envs/internal.yaml b/deploy/helm_charts/envs/internal.yaml
@@ -36,6 +36,14 @@ ingress:
   certName: website-ssl-certificate
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/magic_eye.yaml b/deploy/helm_charts/envs/magic_eye.yaml
@@ -37,6 +37,14 @@ ingress:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/prod.yaml b/deploy/helm_charts/envs/prod.yaml
@@ -41,6 +41,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml
@@ -32,6 +32,14 @@ serviceAccount:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "medium_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null

diff --git a/deploy/helm_charts/envs/unsdg.yaml b/deploy/helm_charts/envs/unsdg.yaml
@@ -40,6 +40,14 @@ ingress:
 
 nl:
   enabled: true
+  embeddingsSpec:
+    defaultIndex: "sdg_ft"
+    enabledIndexes: [
+      "bio_ft"
+      "medium_ft",
+      "sdg_ft",
+      "undata_ft",
+    ]
 
 serviceGroups:
   recon: null