Merge pull request #884 from openzim/not_overhelm

Do not request all schedules details at once, proceed in small batches
openzim · Dec 12, 2023 · 9672e25 · 9672e25
2 parents d25e0f0 + e0be511
commit 9672e25
Show file tree

Hide file tree

Showing 4 changed files with 210 additions and 45 deletions.
diff --git a/dispatcher/frontend-ui/package.json b/dispatcher/frontend-ui/package.json
@@ -11,7 +11,7 @@
     "@fortawesome/fontawesome-svg-core": "^1.2.25",
     "@fortawesome/free-solid-svg-icons": "^5.11.2",
     "@fortawesome/vue-fontawesome": "^0.1.8",
-    "axios": "^0.21.2",
+    "axios": "^1.6.0",
     "bootstrap": "^4.4.1",
     "bootstrap-vue": "^2.1.0",
     "core-js": "^3.3.2",

diff --git a/dispatcher/frontend-ui/src/components/PipelineTable.vue b/dispatcher/frontend-ui/src/components/PipelineTable.vue
@@ -180,12 +180,20 @@
       },
       async loadLastRuns() {
         let parent = this;
+        parent.last_runs_loaded = false;
         let schedule_names = parent.tasks.map(function (item) { return item.schedule_name; }).unique();
 
         parent.toggleLoader("fetching last runs…");
-        let requests = schedule_names.map(function (schedule_name) {
-            return parent.queryAPI('get', "/schedules/" + schedule_name);
-          });
+        const chunkSize = Constants.TASKS_LOAD_SCHEDULES_CHUNK_SIZE;
+        let requests = []
+        for (let i = 0; i < schedule_names.length; i += chunkSize) {
+            const chunk = schedule_names.slice(i, i + chunkSize);
+            for (let iChunk in chunk) {
+              requests.push(parent.queryAPI('get', "/schedules/" + chunk[iChunk]))
+            }
+            await Constants.getDelay(Constants.TASKS_LOAD_SCHEDULES_DELAY)
+        }
+
         const results = await Promise.all(requests.map(p => p.catch(e => e)));
 
         results.forEach(function (response, index) {

diff --git a/dispatcher/frontend-ui/src/constants.js b/dispatcher/frontend-ui/src/constants.js
@@ -295,6 +295,10 @@ function get_timezone_details() {
   return {tz: dt.zoneName, offset: dt.o, offsetstr: offsetstr}
 }
 
+function getDelay(milliseconds) { // retrieve a promise making a pause in milliseconds
+  return new Promise(resolve => setTimeout(resolve, milliseconds));
+}
+
 var DEFAULT_CPU_SHARE = 1024;
 
 var ZIMFARM_WEBAPI = window.environ.ZIMFARM_WEBAPI || "https://api.farm.openzim.org/v1";
@@ -307,6 +311,8 @@ export default {
   },
   zimfarm_webapi: ZIMFARM_WEBAPI,
   kiwix_download_url:  window.environ.ZIMFARM_KIWIX_DOWNLOAD_URL || "https://download.kiwix.org/zim",
+  TASKS_LOAD_SCHEDULES_CHUNK_SIZE: parseInt(window.environ.ZIMFARM_TASKS_LOAD_SCHEDULES_CHUNK_SIZE, 10) || 5,
+  TASKS_LOAD_SCHEDULES_DELAY: parseInt(window.environ.ZIMFARM_TASKS_LOAD_SCHEDULES_DELAY, 10) || 100,
   DEFAULT_CPU_SHARE: DEFAULT_CPU_SHARE,  // used to generate docker cpu-shares
   DEFAULT_FIRE_PRIORITY: 5,
   DEFAULT_LIMIT: 20,
@@ -527,4 +533,5 @@ export default {
   fromSeconds: DateTime.fromSeconds,
   to_timestamp: to_timestamp,
   is_ios_firefox: isFirefoxOnIOS(),
+  getDelay: getDelay,
 };