Merge pull request cdapio#15559 from cdapio/CDAP-20956

[CDAP-20956] increase system worker request retry timeout
cloudsufi · Feb 22, 2024 · 7eea423 · 7eea423
2 parents fa149e1 + fb2f45e
commit 7eea423
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 2 deletions.
diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java b/cdap-common/src/main/java/io/cdap/cdap/common/internal/remote/RemoteTaskExecutor.java
@@ -90,13 +90,15 @@ public RemoteTaskExecutor(CConfiguration cConf, MetricsCollectionService metrics
     this.remoteClient = remoteClientFactory.createRemoteClient(serviceName,
         httpRequestConfig,
         Constants.Gateway.INTERNAL_API_VERSION_3);
-    this.retryStrategy = RetryStrategies.fromConfiguration(cConf,
-        Constants.Service.TASK_WORKER + ".");
     this.metricsCollectionService = metricsCollectionService;
     if (workerType == Type.TASK_WORKER) {
       this.workerUrl = TASK_WORKER_URL;
+      this.retryStrategy = RetryStrategies.fromConfiguration(cConf,
+          Constants.Service.TASK_WORKER + ".");
     } else {
       this.workerUrl = SYSTEM_WORKER_URL;
+      this.retryStrategy = RetryStrategies.fromConfiguration(cConf,
+          Constants.Service.SYSTEM_WORKER + ".");
     }
   }
 

diff --git a/cdap-common/src/main/resources/cdap-default.xml b/cdap-common/src/main/resources/cdap-default.xml
@@ -4251,6 +4251,51 @@
     </description>
   </property>
 
+  <!-- Retry configuration for system worker -->
+  <!-- Use an exponential delay that cap at 2 seconds interval -->
+  <property>
+    <name>system.worker.retry.policy.base.delay.ms</name>
+    <value>100</value>
+    <description>
+      The base delay between retries in milliseconds
+    </description>
+  </property>
+
+  <property>
+    <name>system.worker.retry.policy.max.delay.ms</name>
+    <value>2000</value>
+    <description>
+      The maximum delay between retries in milliseconds
+    </description>
+  </property>
+
+  <!-- Set to Integer max value, so the retry attempts end based on task.worker.retry.policy.max.time.secs -->
+  <property>
+    <name>system.worker.retry.policy.max.retries</name>
+    <value>2147483647</value>
+    <description>
+      The maximum number of retries to attempt before aborting
+    </description>
+  </property>
+
+  <!-- Set to 300s to match the default http request timeout -->
+  <property>
+    <name>system.worker.retry.policy.max.time.secs</name>
+    <value>300</value>
+    <description>
+      The maximum elapsed time in seconds before retries are aborted
+    </description>
+  </property>
+
+  <property>
+    <name>system.worker.retry.policy.type</name>
+    <value>exponential.backoff</value>
+    <description>
+      The type of retry policy for programs. Allowed options:
+      "none", "fixed.delay", or "exponential.backoff".
+    </description>
+  </property>
+
   <!-- Retry configuration for artifact cache -->
   <!-- Use an exponential delay that cap at 2 seconds interval -->
   <property>