load rrdcontext dimensions in batches (netdata#19304)

* load rrdcontext dimensions in batches * Batch load all dimensions of a host --------- Co-authored-by: Stelios Fragkakis <[email protected]>
stelfrag · Dec 31, 2024 · 50b2ade · 50b2ade
1 parent 3ad344a
commit 50b2ade
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 15 deletions.
diff --git a/src/database/contexts/internal.h b/src/database/contexts/internal.h
@@ -443,6 +443,7 @@ void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool
 #define rrdcontext_lock(rc) spinlock_lock(&((rc)->spinlock))
 #define rrdcontext_unlock(rc) spinlock_unlock(&((rc)->spinlock))
 
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
 void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function);
 void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function);
 

diff --git a/src/database/contexts/metric.c b/src/database/contexts/metric.c
@@ -2,7 +2,7 @@
 
 #include "internal.h"
 
-static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
 
 inline const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) {
     RRDMETRIC *rm = rrdmetric_acquired_value(rma);
@@ -225,7 +225,7 @@ void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) {
 }
 
 // trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs
-static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
+void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
     if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)))
         rrdmetric_set_archived(rm);
 

diff --git a/src/database/contexts/worker.c b/src/database/contexts/worker.c
@@ -28,8 +28,26 @@ void load_instance_labels_on_demand(nd_uuid_t *uuid, void *data) {
     ctx_get_label_list(uuid, rrdinstance_load_clabel, data);
 }
 
-static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
-    RRDINSTANCE *ri = data;
+static void rrdinstance_load_dimension_callback(SQL_DIMENSION_DATA *sd, void *data) {
+    RRDHOST *host = data;
+    RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, sd->context);
+    if(!rca) {
+        nd_log(NDLS_DAEMON, NDLP_ERR,
+               "RRDCONTEXT: context '%s' is not found in host '%s'",
+               sd->context, rrdhost_hostname(host));
+        return;
+    }
+    RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
+
+    RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, sd->chart_id);
+    if(!ria) {
+        rrdcontext_release(rca);
+        nd_log(NDLS_DAEMON, NDLP_ERR,
+               "RRDCONTEXT: instance '%s' of context '%s' is not found in host '%s'",
+               sd->chart_id, sd->context, rrdhost_hostname(host));
+        return;
+    }
+    RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
 
     RRDMETRIC trm = {
             .id = string_strdupz(sd->id),
@@ -41,9 +59,12 @@ static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
     uuid_copy(trm.uuid, sd->dim_id);
 
     dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm));
+
+    rrdinstance_release(ria);
+    rrdcontext_release(rca);
 }
 
-static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
+static void rrdinstance_load_instance_callback(SQL_CHART_DATA *sc, void *data) {
     RRDHOST *host = data;
 
     RRDCONTEXT tc = {
@@ -74,10 +95,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
     uuid_copy(tri.uuid, sc->chart_id);
 
     RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, sc->id, &tri, sizeof(tri));
-    RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
 
-    ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
-    rrdinstance_trigger_updates(ri, __FUNCTION__ );
     rrdinstance_release(ria);
     rrdcontext_release(rca);
 }
@@ -106,12 +124,23 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {
         return;
 
     ctx_get_context_list(&host->host_id.uuid, rrdcontext_load_context_callback, host);
-    ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_chart_callback, host);
+    ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_instance_callback, host);
+    ctx_get_dimension_list(&host->host_id.uuid, rrdinstance_load_dimension_callback, host);
 
     RRDCONTEXT *rc;
     dfe_start_read(host->rrdctx.contexts, rc) {
-                rrdcontext_trigger_updates(rc, __FUNCTION__ );
+        RRDINSTANCE *ri;
+        dfe_start_read(rc->rrdinstances, ri) {
+            RRDMETRIC *rm;
+            dfe_start_read(ri->rrdmetrics, rm) {
+                rrdmetric_trigger_updates(rm, __FUNCTION__ );
             }
+            dfe_done(rm);
+            rrdinstance_trigger_updates(ri, __FUNCTION__ );
+        }
+        dfe_done(ri);
+        rrdcontext_trigger_updates(rc, __FUNCTION__ );
+    }
     dfe_done(rc);
 
     rrdcontext_garbage_collect_single_host(host, false);

diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c
@@ -115,17 +115,17 @@ void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *,
 }
 
 // Dimension list
-#define CTX_GET_DIMENSION_LIST  "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \
-    "FROM dimension d WHERE d.chart_id = @id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"
-void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
+#define CTX_GET_DIMENSION_LIST  "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context " \
+    "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC"
+void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data)
 {
     static __thread sqlite3_stmt *res = NULL;
 
     if (!PREPARE_COMPILED_STATEMENT(db_meta, CTX_GET_DIMENSION_LIST, &res))
         return;
 
     int param = 0;
-    SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC));
+    SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC));
 
     SQL_DIMENSION_DATA dimension_data;
 
@@ -135,6 +135,8 @@ void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION
         dimension_data.id = (char *) sqlite3_column_text(res, 1);
         dimension_data.name = (char *) sqlite3_column_text(res, 2);
         dimension_data.hidden = sqlite3_column_int(res, 3);
+        dimension_data.chart_id = (char *) sqlite3_column_text(res, 4);
+        dimension_data.context = (char *) sqlite3_column_text(res, 5);
         dict_cb(&dimension_data, data);
     }
 

diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h
@@ -25,6 +25,9 @@ typedef struct ctx_dimension {
     char *id;
     char *name;
     bool hidden;
+
+    char *context;
+    char *chart_id;
 } SQL_DIMENSION_DATA;
 
 typedef struct ctx_label {
@@ -56,7 +59,7 @@ void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEX
 
 void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data);
 void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data);
-void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data);
+void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data);
 
 int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);