From 50b2adeb39781bf6779560a254319604be0657ee Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 31 Dec 2024 17:00:58 +0000 Subject: [PATCH] load rrdcontext dimensions in batches (#19304) * load rrdcontext dimensions in batches * Batch load all dimensions of a host --------- Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> --- src/database/contexts/internal.h | 1 + src/database/contexts/metric.c | 4 +-- src/database/contexts/worker.c | 45 +++++++++++++++++++++++----- src/database/sqlite/sqlite_context.c | 10 ++++--- src/database/sqlite/sqlite_context.h | 5 +++- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/database/contexts/internal.h b/src/database/contexts/internal.h index b0b2c45796c009..cea5fb37075ba5 100644 --- a/src/database/contexts/internal.h +++ b/src/database/contexts/internal.h @@ -443,6 +443,7 @@ void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool #define rrdcontext_lock(rc) spinlock_lock(&((rc)->spinlock)) #define rrdcontext_unlock(rc) spinlock_unlock(&((rc)->spinlock)) +void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function); void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function); void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function); diff --git a/src/database/contexts/metric.c b/src/database/contexts/metric.c index 2808ac2f046853..c65d0626fec1c2 100644 --- a/src/database/contexts/metric.c +++ b/src/database/contexts/metric.c @@ -2,7 +2,7 @@ #include "internal.h" -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function); +void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function); inline const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma) { RRDMETRIC *rm = rrdmetric_acquired_value(rma); @@ -225,7 +225,7 @@ void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) { } // trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs -static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) { +void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) { if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD))) rrdmetric_set_archived(rm); diff --git a/src/database/contexts/worker.c b/src/database/contexts/worker.c index ea1bda1cc555ae..6747810a8f7dae 100644 --- a/src/database/contexts/worker.c +++ b/src/database/contexts/worker.c @@ -28,8 +28,26 @@ void load_instance_labels_on_demand(nd_uuid_t *uuid, void *data) { ctx_get_label_list(uuid, rrdinstance_load_clabel, data); } -static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) { - RRDINSTANCE *ri = data; +static void rrdinstance_load_dimension_callback(SQL_DIMENSION_DATA *sd, void *data) { + RRDHOST *host = data; + RRDCONTEXT_ACQUIRED *rca = (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item(host->rrdctx.contexts, sd->context); + if(!rca) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "RRDCONTEXT: context '%s' is not found in host '%s'", + sd->context, rrdhost_hostname(host)); + return; + } + RRDCONTEXT *rc = rrdcontext_acquired_value(rca); + + RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_get_and_acquire_item(rc->rrdinstances, sd->chart_id); + if(!ria) { + rrdcontext_release(rca); + nd_log(NDLS_DAEMON, NDLP_ERR, + "RRDCONTEXT: instance '%s' of context '%s' is not found in host '%s'", + sd->chart_id, sd->context, rrdhost_hostname(host)); + return; + } + RRDINSTANCE *ri = rrdinstance_acquired_value(ria); RRDMETRIC trm = { .id = string_strdupz(sd->id), @@ -41,9 +59,12 @@ static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) { uuid_copy(trm.uuid, sd->dim_id); dictionary_set(ri->rrdmetrics, string2str(trm.id), &trm, sizeof(trm)); + + rrdinstance_release(ria); + rrdcontext_release(rca); } -static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) { +static void rrdinstance_load_instance_callback(SQL_CHART_DATA *sc, void *data) { RRDHOST *host = data; RRDCONTEXT tc = { @@ -74,10 +95,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) { uuid_copy(tri.uuid, sc->chart_id); RRDINSTANCE_ACQUIRED *ria = (RRDINSTANCE_ACQUIRED *)dictionary_set_and_acquire_item(rc->rrdinstances, sc->id, &tri, sizeof(tri)); - RRDINSTANCE *ri = rrdinstance_acquired_value(ria); - ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri); - rrdinstance_trigger_updates(ri, __FUNCTION__ ); rrdinstance_release(ria); rrdcontext_release(rca); } @@ -106,12 +124,23 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) { return; ctx_get_context_list(&host->host_id.uuid, rrdcontext_load_context_callback, host); - ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_chart_callback, host); + ctx_get_chart_list(&host->host_id.uuid, rrdinstance_load_instance_callback, host); + ctx_get_dimension_list(&host->host_id.uuid, rrdinstance_load_dimension_callback, host); RRDCONTEXT *rc; dfe_start_read(host->rrdctx.contexts, rc) { - rrdcontext_trigger_updates(rc, __FUNCTION__ ); + RRDINSTANCE *ri; + dfe_start_read(rc->rrdinstances, ri) { + RRDMETRIC *rm; + dfe_start_read(ri->rrdmetrics, rm) { + rrdmetric_trigger_updates(rm, __FUNCTION__ ); } + dfe_done(rm); + rrdinstance_trigger_updates(ri, __FUNCTION__ ); + } + dfe_done(ri); + rrdcontext_trigger_updates(rc, __FUNCTION__ ); + } dfe_done(rc); rrdcontext_garbage_collect_single_host(host, false); diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c index 1d0c768e5b5ea0..8207c64a1c7bcb 100644 --- a/src/database/sqlite/sqlite_context.c +++ b/src/database/sqlite/sqlite_context.c @@ -115,9 +115,9 @@ void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, } // Dimension list -#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END " \ - "FROM dimension d WHERE d.chart_id = @id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC" -void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data) +#define CTX_GET_DIMENSION_LIST "SELECT d.dim_id, d.id, d.name, CASE WHEN INSTR(d.options,\"hidden\") > 0 THEN 1 ELSE 0 END, c.type||'.'||c.id, c.context " \ + "FROM dimension d, chart c WHERE c.host_id = @host_id AND d.chart_id = c.chart_id AND d.dim_id IS NOT NULL ORDER BY d.rowid ASC" +void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data) { static __thread sqlite3_stmt *res = NULL; @@ -125,7 +125,7 @@ void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION return; int param = 0; - SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC)); + SQLITE_BIND_FAIL(done, sqlite3_bind_blob(res, ++param, host_uuid, sizeof(*host_uuid), SQLITE_STATIC)); SQL_DIMENSION_DATA dimension_data; @@ -135,6 +135,8 @@ void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION dimension_data.id = (char *) sqlite3_column_text(res, 1); dimension_data.name = (char *) sqlite3_column_text(res, 2); dimension_data.hidden = sqlite3_column_int(res, 3); + dimension_data.chart_id = (char *) sqlite3_column_text(res, 4); + dimension_data.context = (char *) sqlite3_column_text(res, 5); dict_cb(&dimension_data, data); } diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h index ce2fc31b129742..614f0ef329665c 100644 --- a/src/database/sqlite/sqlite_context.h +++ b/src/database/sqlite/sqlite_context.h @@ -25,6 +25,9 @@ typedef struct ctx_dimension { char *id; char *name; bool hidden; + + char *context; + char *chart_id; } SQL_DIMENSION_DATA; typedef struct ctx_label { @@ -56,7 +59,7 @@ void ctx_get_context_list(nd_uuid_t *host_uuid, void (*dict_cb)(VERSIONED_CONTEX void ctx_get_chart_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_CHART_DATA *, void *), void *data); void ctx_get_label_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_CLABEL_DATA *, void *), void *data); -void ctx_get_dimension_list(nd_uuid_t *chart_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data); +void ctx_get_dimension_list(nd_uuid_t *host_uuid, void (*dict_cb)(SQL_DIMENSION_DATA *, void *), void *data); int ctx_store_context(nd_uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);