feat(throttle): add pending throttling controls for v0.14.2 (#510)

acryldata · Oct 10, 2024 · 940132e · 940132e
1 parent e7a3562
commit 940132e
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 5 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,13 +4,13 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.29
+version: 0.4.30
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1
 dependencies:
   - name: datahub-gms
-    version: 0.2.173
+    version: 0.2.174
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
@@ -22,7 +22,7 @@ dependencies:
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.164
+    version: 0.2.165
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron

diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.173
+version: 0.2.174
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml
@@ -401,6 +401,10 @@ spec:
             - name: MCP_THROTTLE_UPDATE_INTERVAL_MS
               value: {{ . | quote }}
             {{- end }}
+            {{- with .apiRequests.enabled }}
+            - name: MCP_API_REQUESTS_THROTTLE_ENABLED
+              value: {{ . | quote }}
+            {{- end }}
             {{- if .versioned.enabled }}
             - name: MCP_VERSIONED_THROTTLE_ENABLED
               value: "true"

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.164
+version: 0.2.165
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml
@@ -278,6 +278,10 @@ spec:
             - name: MCP_THROTTLE_UPDATE_INTERVAL_MS
               value: {{ . | quote }}
             {{- end }}
+            {{- with .mceConsumer.enabled }}
+            - name: MCP_MCE_CONSUMER_THROTTLE_ENABLED
+              value: {{ . | quote }}
+            {{- end }}
             {{- if .versioned.enabled }}
             - name: MCP_VERSIONED_THROTTLE_ENABLED
               value: "true"

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -811,6 +811,10 @@ global:
     mcp:
       throttle:
         # updateIntervalMs: 60000
+        mceConsumer:  # v0.14.2+
+          enabled: false
+        apiRequests:  # v0.14.2+
+          enabled: false
         ## Versioned MCL topic
         versioned:
           ## Whether to throttle MCP processing based on MCL backlog