Improve shutdown when collectors are active (netdata#16315)

* Collectors should not be running at this point, but allow shutdown to continue after several retries (workaround) * Proceed with shutdown after 10 attempts
thiagoftsm · Nov 1, 2023 · ca592a9 · ca592a9
1 parent 9d16789
commit ca592a9
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/daemon/main.c b/daemon/main.c
@@ -440,7 +440,8 @@ void netdata_cleanup_and_exit(int ret) {
             delta_shutdown_time("wait for dbengine collectors to finish");
 
             size_t running = 1;
-            while(running) {
+            size_t count = 10;
+            while(running && count) {
                 running = 0;
                 for (size_t tier = 0; tier < storage_tiers; tier++)
                     running += rrdeng_collectors_running(multidb_ctx[tier]);
@@ -451,6 +452,7 @@ void netdata_cleanup_and_exit(int ret) {
                     // sleep_usec(100 * USEC_PER_MS);
                     cleanup_destroyed_dictionaries();
                 }
+                count--;
             }
 
             delta_shutdown_time("wait for dbengine main cache to finish flushing");

diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
@@ -1241,12 +1241,14 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
     // 4. then wait for completion
 
     bool logged = false;
-    while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && !unittest_running) {
+    size_t count = 10;
+    while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && count && !unittest_running) {
         if(!logged) {
             netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier);
             logged = true;
         }
         sleep_usec(100 * USEC_PER_MS);
+        count--;
     }
 
     netdata_log_info("DBENGINE: flushing main cache for tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier);