diff --git a/src/daemon/main.c b/src/daemon/main.c index 46f008cb56f106..d1e795f2313e63 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -448,11 +448,10 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re #endif } - sql_close_context_database(); - watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB); + sqlite_close_databases(); + watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_DATABASES); + sqlite_library_shutdown(); - sql_close_database(); - watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB); // unlink the pid if(pidfile[0]) { @@ -1500,22 +1499,24 @@ int main(int argc, char **argv) { #endif if(strcmp(optarg, "sqlite-meta-recover") == 0) { - sql_init_database(DB_CHECK_RECOVER, 0); + sql_init_meta_database(DB_CHECK_RECOVER, 0); return 0; } if(strcmp(optarg, "sqlite-compact") == 0) { - sql_init_database(DB_CHECK_RECLAIM_SPACE, 0); + sql_init_meta_database(DB_CHECK_RECLAIM_SPACE, 0); return 0; } if(strcmp(optarg, "sqlite-analyze") == 0) { - sql_init_database(DB_CHECK_ANALYZE, 0); + sql_init_meta_database(DB_CHECK_ANALYZE, 0); return 0; } if(strcmp(optarg, "unittest") == 0) { unittest_running = true; + if (sqlite_library_init()) + return 1; if (pluginsd_parser_unittest()) return 1; if (unit_test_static_threads()) return 1; @@ -1539,6 +1540,7 @@ int main(int argc, char **argv) { if (ctx_unittest()) return 1; if (uuid_unittest()) return 1; if (dyncfg_unittest()) return 1; + sqlite_library_shutdown(); fprintf(stderr, "\n\nALL TESTS PASSED\n\n"); return 0; } @@ -2054,6 +2056,8 @@ int main(int argc, char **argv) { exit(1); } } + if (sqlite_library_init()) + fatal("Failed to initialize sqlite library"); // -------------------------------------------------------------------- // Initialize ML configuration diff --git a/src/daemon/unit_test.c b/src/daemon/unit_test.c index 1b350fecd3bd7c..183770baf4d7f4 100644 --- a/src/daemon/unit_test.c +++ b/src/daemon/unit_test.c @@ -2546,7 +2546,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n"); - (void) sql_init_database(DB_CHECK_NONE, 1); + (void)sql_init_meta_database(DB_CHECK_NONE, 1); host = dbengine_rrdhost_find_or_create("dbengine-stress-test"); if (NULL == host) return; diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c index e19d9e2d6f1384..3eea22019d60e4 100644 --- a/src/daemon/watcher.c +++ b/src/daemon/watcher.c @@ -85,8 +85,7 @@ void *watcher_main(void *arg) watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH); watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING); watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS); - watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB); - watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB); + watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES); watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE); watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES); watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE); @@ -146,10 +145,8 @@ void watcher_thread_start() { "wait for dbengine main cache to finish flushing"; watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg = "stop dbengine tiers"; - watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg = - "close SQL context db"; - watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg = - "close SQL main db"; + watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_DATABASES].msg = + "close SQL databases"; watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg = "remove pid file"; watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg = diff --git a/src/daemon/watcher.h b/src/daemon/watcher.h index 6c9ce155238fc7..b785ca436288a7 100644 --- a/src/daemon/watcher.h +++ b/src/daemon/watcher.h @@ -27,8 +27,7 @@ typedef enum { WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH, WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING, WATCHER_STEP_ID_STOP_DBENGINE_TIERS, - WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB, - WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB, + WATCHER_STEP_ID_CLOSE_SQL_DATABASES, WATCHER_STEP_ID_REMOVE_PID_FILE, WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES, WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE, diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c index bb120134235680..9bf4853a38fe06 100644 --- a/src/database/rrdhost.c +++ b/src/database/rrdhost.c @@ -1009,7 +1009,7 @@ void dbengine_init(char *hostname) { int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) { rrdhost_init(); - if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) { + if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) { if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { set_late_global_environment(system_info); fatal("Failed to initialize SQLite"); diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c index 683ed12cd383ee..ad76a1ee2c4441 100644 --- a/src/database/sqlite/sqlite_context.c +++ b/src/database/sqlite/sqlite_context.c @@ -71,24 +71,6 @@ int sql_init_context_database(int memory) return 0; } -/* - * Close the sqlite database - */ - -void sql_close_context_database(void) -{ - int rc; - if (unlikely(!db_context_meta)) - return; - - netdata_log_info("Closing context SQLite database"); - - rc = sqlite3_close_v2(db_context_meta); - if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc)); - db_context_meta = NULL; -} - // // Fetching data // @@ -422,6 +404,12 @@ int sql_context_cache_stats(int op) return count; } + +uint64_t sqlite_get_context_space(void) +{ + return sqlite_get_db_space(db_context_meta); +} + // // TESTING FUNCTIONS // @@ -456,7 +444,8 @@ int ctx_unittest(void) uuid_t host_uuid; uuid_generate(host_uuid); - initialize_thread_key_pool(); + if (sqlite_library_init()) + return 1; int rc = sql_init_context_database(1); @@ -532,7 +521,8 @@ int ctx_unittest(void) ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); netdata_log_info("List context end after delete"); - sql_close_context_database(); + sql_close_database(db_context_meta, "CONTEXT"); + sqlite_library_shutdown(); return 0; } diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h index 2586916eaeec3c..92d02fdd2203b5 100644 --- a/src/database/sqlite/sqlite_context.h +++ b/src/database/sqlite/sqlite_context.h @@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data); int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data); int sql_init_context_database(int memory); +uint64_t sqlite_get_context_space(void); void sql_close_context_database(void); int ctx_unittest(void); #endif //NETDATA_SQLITE_CONTEXT_H diff --git a/src/database/sqlite/sqlite_functions.c b/src/database/sqlite/sqlite_functions.c index 2b01ce79c68feb..1dc2022b37a205 100644 --- a/src/database/sqlite/sqlite_functions.c +++ b/src/database/sqlite/sqlite_functions.c @@ -1,94 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "sqlite_functions.h" -#include "sqlite3recover.h" -#include "sqlite_db_migration.h" - -#define DB_METADATA_VERSION 16 - -const char *database_config[] = { - "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, " - "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, " - "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default ''," - "hops INT NOT NULL DEFAULT 0," - "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0," - "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', " - "entries INT NOT NULL DEFAULT 0," - "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)", - - "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, " - "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, " - "chart_type int, memory_mode int, history_entries)", - - "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, " - "multiplier int, divisor int , algorithm int, options text)", - - "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)", - - "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)", - - "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)", - - "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " - "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))", - - "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)", - - "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, " - "on_key text, class text, component text, type text, os text, hosts text, lookup text, " - "every text, units text, calc text, families text, plugin text, module text, charts text, green text, " - "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, " - "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, " - "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text)", - - "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, " - "date_created INT, PRIMARY KEY(host_id, system_key))", - - "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, " - "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))", - - "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)" - " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END", - - "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, " - "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, " - "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))", - - "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)", - - "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, " - "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, " - "flags int, exec_run_timestamp int, delay_up_to_timestamp int, " - "info text, exec_code int, new_status real, old_status real, delay int, " - "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)", - - "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)", - "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)", - "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)", - "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)", - "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)", - "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)", - - NULL -}; - -const char *database_cleanup[] = { - "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)", - "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)", - "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)", - "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)", - "DROP TRIGGER IF EXISTS tr_dim_del", - "DROP INDEX IF EXISTS ind_d1", - "DROP INDEX IF EXISTS ind_c1", - "DROP INDEX IF EXISTS ind_c2", - "DROP INDEX IF EXISTS alert_hash_index", - "DROP INDEX IF EXISTS health_log_d_ind_4", - "DROP INDEX IF EXISTS health_log_d_ind_1", - "DROP INDEX IF EXISTS health_log_d_ind_5", - NULL -}; - -sqlite3 *db_meta = NULL; #define MAX_PREPARED_STATEMENTS (32) pthread_key_t key_pool[MAX_PREPARED_STATEMENTS]; @@ -152,45 +64,6 @@ static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database) return false; } -static void recover_database(const char *sqlite_database, const char *new_sqlite_database) -{ - sqlite3 *database; - int rc = sqlite3_open(sqlite_database, &database); - if (rc != SQLITE_OK) - return; - - netdata_log_info("Recover %s", sqlite_database); - netdata_log_info(" to %s", new_sqlite_database); - - // This will remove the -shm and -wal files when we close the database - (void) db_execute(database, "select count(*) from sqlite_master limit 0"); - - sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database); - if (recover) { - - rc = sqlite3_recover_run(recover); - - if (rc == SQLITE_OK) - netdata_log_info("Recover complete"); - else - netdata_log_info("Recover encountered an error but the database may be usable"); - - rc = sqlite3_recover_finish(recover); - - (void) sqlite3_close(database); - - if (rc == SQLITE_OK) { - rc = rename(new_sqlite_database, sqlite_database); - if (rc == 0) { - netdata_log_info("Renamed %s", new_sqlite_database); - netdata_log_info(" to %s", sqlite_database); - } - } - } - else - (void) sqlite3_close(database); -} - int execute_insert(sqlite3_stmt *res) { int rc; @@ -279,14 +152,11 @@ static void add_stmt_to_list(sqlite3_stmt *res) static void release_statement(void *statement) { int rc; -#ifdef NETDATA_DEV_MODE - netdata_log_info("Thread %d: Cleaning prepared statement on %p", gettid(), statement); -#endif if (unlikely(rc = sqlite3_finalize((sqlite3_stmt *) statement) != SQLITE_OK)) error_report("Failed to finalize statement, rc = %d", rc); } -void initialize_thread_key_pool(void) +static void initialize_thread_key_pool(void) { for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++) (void)pthread_key_create(&key_pool[i], release_statement); @@ -303,13 +173,9 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state key = &key_pool[keys_used++]; int rc = sqlite3_prepare_v2(database, query, -1, statement, 0); - if (likely(rc == SQLITE_OK)) { - if (likely(key)) { + if (rc == SQLITE_OK) { + if (key) ret = pthread_setspecific(*key, *statement); -#ifdef NETDATA_DEV_MODE - netdata_log_info("Thread %d: Using key %u on statement %p", gettid(), keys_used, *statement); -#endif - } if (ret) add_stmt_to_list(*statement); } @@ -355,202 +221,6 @@ int init_database_batch(sqlite3 *database, const char *batch[], const char *desc return 0; } -static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv) -{ - uuid_t uuid; - - if ( argc != 1 ){ - sqlite3_result_null(context); - return ; - } - int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid); - if (rc == -1) { - sqlite3_result_null(context); - return ; - } - - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); -} - -void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv) -{ - if (argc != 1 ){ - sqlite3_result_null(context); - return ; - } - - if (sqlite3_value_int(argv[0]) != 0) { - struct timespec req = {.tv_sec = 0, .tv_nsec = 1}; - nanosleep(&req, NULL); - } - - sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec()); -} - -void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv) -{ - (void)argc; - (void)argv; - - uuid_t uuid; - uuid_generate_random(uuid); - sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); -} - -/* - * Initialize the SQLite database - * Return 0 on success - */ -int sql_init_database(db_check_action_type_t rebuild, int memory) -{ - char *err_msg = NULL; - char sqlite_database[FILENAME_MAX + 1]; - int rc; - - if (likely(!memory)) { - snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir); - rc = unlink(sqlite_database); - snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir); - - if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) { - char new_sqlite_database[FILENAME_MAX + 1]; - snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir); - recover_database(sqlite_database, new_sqlite_database); - if (rebuild & DB_CHECK_RECOVER) - return 0; - } - } - else - strcpy(sqlite_database, ":memory:"); - - rc = sqlite3_open(sqlite_database, &db_meta); - if (rc != SQLITE_OK) { - error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); - char *error_str = get_database_extented_error(db_meta, 0, "meta_open"); - if (error_str) - analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); - freez(error_str); - sqlite3_close(db_meta); - db_meta = NULL; - return 1; - } - - if (rebuild & DB_CHECK_RECLAIM_SPACE) { - netdata_log_info("Reclaiming space of %s", sqlite_database); - rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg); - if (rc != SQLITE_OK) { - error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg); - sqlite3_free(err_msg); - } - else { - (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); - (void) sqlite3_close(db_meta); - } - return 1; - } - - if (rebuild & DB_CHECK_ANALYZE) { - errno = 0; - netdata_log_info("Running ANALYZE on %s", sqlite_database); - rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg); - if (rc != SQLITE_OK) { - error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg); - sqlite3_free(err_msg); - } - else { - (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); - (void) sqlite3_close(db_meta); - } - return 1; - } - - netdata_log_info("SQLite database %s initialization", sqlite_database); - - rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to register internal u2h function"); - - rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to register internal now_usec function"); - - rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to register internal uuid_random function"); - - int target_version = DB_METADATA_VERSION; - - if (likely(!memory)) - target_version = perform_database_migration(db_meta, DB_METADATA_VERSION); - - if (configure_sqlite_database(db_meta, target_version, "meta_config")) - return 1; - - if (init_database_batch(db_meta, &database_config[0], "meta_init")) - return 1; - - if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup")) - return 1; - - netdata_log_info("SQLite database initialization completed"); - - initialize_thread_key_pool(); - - return 0; -} - -/* - * Close the sqlite database - */ - -void sql_close_database(void) -{ - int rc; - if (unlikely(!db_meta)) - return; - - netdata_log_info("Closing SQLite database"); - - add_stmt_to_list(NULL); - - (void) db_execute(db_meta, "PRAGMA analysis_limit=10000"); - (void) db_execute(db_meta, "PRAGMA optimize"); - - rc = sqlite3_close_v2(db_meta); - if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc)); - db_meta = NULL; -} - -int exec_statement_with_uuid(const char *sql, uuid_t *uuid) -{ - int rc, result = 1; - sqlite3_stmt *res = NULL; - - rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement %s, rc = %d", sql, rc); - return 1; - } - - rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc); - goto skip; - } - - rc = execute_insert(res); - if (likely(rc == SQLITE_DONE)) - result = SQLITE_OK; - else - error_report("Failed to execute %s, rc = %d", sql, rc); - -skip: - rc = sqlite3_finalize(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to finalize statement %s, rc = %d", sql, rc); - return result; -} // Return 0 OK // Return 1 Failed int db_execute(sqlite3 *db, const char *cmd) @@ -580,376 +250,124 @@ int db_execute(sqlite3 *db, const char *cmd) return (rc != SQLITE_OK); } -static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) +// Utils +int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null) { - if (unlikely(!host)) - return; - - if (unlikely(!node_id)) { - freez(host->node_id); - __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED); - return; - } - - struct aclk_sync_cfg_t *wc = host->aclk_config; - - if (unlikely(!host->node_id)) { - uuid_t *t = mallocz(sizeof(*host->node_id)); - uuid_copy(*t, *node_id); - __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED); - } - else { - uuid_copy(*(host->node_id), *node_id); - } - - if (unlikely(!wc)) - sql_create_aclk_table(host, &host->host_uuid, node_id); - else - uuid_unparse_lower(*node_id, wc->node_id); + if (likely(text)) + return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC); + if (!can_be_null) + return 1; + return sqlite3_bind_null(res, position); } -#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id" +#define SQL_DROP_TABLE "DROP table %s" -int update_node_id(uuid_t *host_id, uuid_t *node_id) +void sql_drop_table(const char *table) { - sqlite3_stmt *res = NULL; - RRDHOST *host = NULL; - int rc = 2; - - char host_guid[GUID_LEN + 1]; - uuid_unparse_lower(*host_id, host_guid); - rrd_wrlock(); - host = rrdhost_find_by_guid(host_guid); - if (likely(host)) - set_host_node_id(host, node_id); - rrd_unlock(); - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return 1; - } - - rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to store node instance information"); - return 1; - } + if (!table) + return; - rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter to store node instance information"); - goto failed; - } + char wstr[255]; + snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table); - rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter to store node instance information"); - goto failed; + int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL); + if (rc != SQLITE_OK) { + error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc); } - - rc = execute_insert(res); - if (unlikely(rc != SQLITE_DONE)) - error_report("Failed to store node instance information, rc = %d", rc); - rc = sqlite3_changes(db_meta); - -failed: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when storing node instance information"); - - return rc - 1; } -#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL" - -int get_node_id(uuid_t *host_id, uuid_t *node_id) +static int get_pragma_value(sqlite3 *database, const char *sql) { sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return 1; - } - - rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to select node instance information for a host"); - return 1; - } - - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter to select node instance information"); - goto failed; - } + int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) + return -1; + int result = -1; rc = sqlite3_step_monitored(res); - if (likely(rc == SQLITE_ROW && node_id)) - uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0))); + if (likely(rc == SQLITE_ROW)) + result = sqlite3_column_int(res, 0); -failed: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when selecting node instance information"); + rc = sqlite3_finalize(res); + (void) rc; - return (rc == SQLITE_ROW) ? 0 : -1; + return result; } -#define SQL_INVALIDATE_NODE_INSTANCES \ - "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \ - "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))" - -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id) +int get_free_page_count(sqlite3 *database) { - sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return; - } - - rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to invalidate node instance ids"); - return; - } - - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter to invalidate node instance information"); - goto failed; - } - - if (claim_id) - rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC); - else - rc = sqlite3_bind_null(res, 2); - - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind claim_id parameter to invalidate node instance information"); - goto failed; - } - - rc = execute_insert(res); - if (unlikely(rc != SQLITE_DONE)) - error_report("Failed to invalidate node instance information, rc = %d", rc); - -failed: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when invalidating node instance information"); + return get_pragma_value(database, "PRAGMA freelist_count"); } -#define SQL_GET_NODE_INSTANCE_LIST \ - "SELECT ni.node_id, ni.host_id, h.hostname " \ - "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0" - -struct node_instance_list *get_node_list(void) +int get_database_page_count(sqlite3 *database) { - struct node_instance_list *node_list = NULL; - sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return NULL; - } - - rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to get node instance information"); - return NULL; - } - - int row = 0; - char host_guid[37]; - while (sqlite3_step_monitored(res) == SQLITE_ROW) - row++; - - if (sqlite3_reset(res) != SQLITE_OK) { - error_report("Failed to reset the prepared statement while fetching node instance information"); - goto failed; - } - node_list = callocz(row + 1, sizeof(*node_list)); - int max_rows = row; - row = 0; - // TODO: Check to remove lock - rrd_rdlock(); - while (sqlite3_step_monitored(res) == SQLITE_ROW) { - if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t)) - uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0))); - if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) { - uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1); - uuid_unparse_lower(*host_id, host_guid); - RRDHOST *host = rrdhost_find_by_guid(host_guid); - if (!host) - continue; - if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) { - netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host)); - continue; - } - uuid_copy(node_list[row].host_id, *host_id); - node_list[row].queryable = 1; - node_list[row].live = (host && (host == localhost || host->receiver - || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0; - node_list[row].hops = (host && host->system_info) ? host->system_info->hops : - uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0; - node_list[row].hostname = - sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL; - } - row++; - if (row == max_rows) - break; - } - rrd_unlock(); - -failed: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when fetching node instance information"); - - return node_list; + return get_pragma_value(database, "PRAGMA page_count"); } -#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id" - -void sql_load_node_id(RRDHOST *host) +uint64_t sqlite_get_db_space(sqlite3 *db) { - sqlite3_stmt *res = NULL; - int rc; - - if (unlikely(!db_meta)) { - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) - error_report("Database has not been initialized"); - return; - } - - rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to fetch node id"); - return; - } - - rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter to load node instance information"); - goto failed; - } + if (!db) + return 0; - rc = sqlite3_step_monitored(res); - if (likely(rc == SQLITE_ROW)) { - if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t))) - set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0)); - else - set_host_node_id(host, NULL); - } + uint64_t page_size = (uint64_t) get_pragma_value(db, "PRAGMA page_size"); + uint64_t page_count = (uint64_t) get_pragma_value(db, "PRAGMA page_count"); -failed: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when loading node instance information"); + return page_size * page_count; } -#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id" +/* + * Close the sqlite database + */ -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info) +void sql_close_database(sqlite3 *database, const char *database_name) { int rc; - - sqlite3_stmt *res = NULL; - - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to read host information"); + if (unlikely(!database)) return; - } - - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host parameter host information"); - goto skip; - } - - while (sqlite3_step_monitored(res) == SQLITE_ROW) { - rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0), - (char *) sqlite3_column_text(res, 1)); - } - -skip: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when reading host information"); -} - -#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \ - "AND label_key IS NOT NULL AND label_value IS NOT NULL" - -RRDLABELS *sql_load_host_labels(uuid_t *host_id) -{ - int rc; - RRDLABELS *labels = NULL; - sqlite3_stmt *res = NULL; + (void) db_execute(database, "PRAGMA analysis_limit=10000"); + (void) db_execute(database, "PRAGMA optimize"); - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to read host information"); - return NULL; - } + netdata_log_info("%s: Closing sqlite database", database_name); - rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host parameter host information"); - goto skip; - } +#ifdef NETDATA_DEV_MODE + int t_count_used,t_count_hit,t_count_miss,t_count_full, dummy; + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_USED, &dummy, &t_count_used, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_HIT, &dummy,&t_count_hit, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &dummy,&t_count_miss, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &dummy,&t_count_full, 0); - labels = rrdlabels_create(); + netdata_log_info("%s: Database lookaside allocation statistics: Used slots %d, Hit %d, Misses due to small slot size %d, Misses due to slots full %d", database_name, + t_count_used,t_count_hit, t_count_miss, t_count_full); - while (sqlite3_step_monitored(res) == SQLITE_ROW) { - rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2)); - } + (void) sqlite3_db_release_memory(database); +#endif -skip: - if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) - error_report("Failed to finalize the prepared statement when reading host information"); - return labels; + rc = sqlite3_close_v2(database); + if (unlikely(rc != SQLITE_OK)) + error_report("%s: Error while closing the sqlite database: rc %d, error \"%s\"", database_name, rc, sqlite3_errstr(rc)); } -// Utils -int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null) +extern sqlite3 *db_context_meta; + +void sqlite_close_databases(void) { - if (likely(text)) - return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC); - if (!can_be_null) - return 1; - return sqlite3_bind_null(res, position); + add_stmt_to_list(NULL); + + sql_close_database(db_context_meta, "CONTEXT"); + sql_close_database(db_meta, "METADATA"); } -int sql_metadata_cache_stats(int op) +int sqlite_library_init(void) { - int count, dummy; + initialize_thread_key_pool(); - if (unlikely(!db_meta)) - return 0; + int rc = sqlite3_initialize(); - netdata_thread_disable_cancelability(); - sqlite3_db_status(db_meta, op, &count, &dummy, 0); - netdata_thread_enable_cancelability(); - return count; + return (SQLITE_OK != rc); } -#define SQL_DROP_TABLE "DROP table %s" - -void sql_drop_table(const char *table) +void sqlite_library_shutdown(void) { - if (!table) - return; - - char wstr[255]; - snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table); - - int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL); - if (rc != SQLITE_OK) { - error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc); - } + (void) sqlite3_shutdown(); } diff --git a/src/database/sqlite/sqlite_functions.h b/src/database/sqlite/sqlite_functions.h index 8e9bdd66bf6ed9..2841b27f6ddc15 100644 --- a/src/database/sqlite/sqlite_functions.h +++ b/src/database/sqlite/sqlite_functions.h @@ -8,36 +8,9 @@ void analytics_set_data_str(char **name, const char *value); -// return a node list -struct node_instance_list { - uuid_t node_id; - uuid_t host_id; - char *hostname; - int live; - int queryable; - int hops; -}; - -typedef enum db_check_action_type { - DB_CHECK_NONE = (1 << 0), - DB_CHECK_RECLAIM_SPACE = (1 << 1), - DB_CHECK_ANALYZE = (1 << 2), - DB_CHECK_CONT = (1 << 3), - DB_CHECK_RECOVER = (1 << 4), -} db_check_action_type_t; - #define SQL_MAX_RETRY (100) #define SQLITE_INSERT_DELAY (10) // Insert delay in case of lock -#define CHECK_SQLITE_CONNECTION(db_meta) \ - if (unlikely(!db_meta)) { \ - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) { \ - return 1; \ - } \ - error_report("Database has not been initialized"); \ - return 1; \ - } - SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt); SQLITE_API int sqlite3_exec_monitored( sqlite3 *db, /* An open database */ @@ -49,35 +22,26 @@ SQLITE_API int sqlite3_exec_monitored( // Initialization and shutdown int init_database_batch(sqlite3 *database, const char *batch[], const char *description); -int sql_init_database(db_check_action_type_t rebuild, int memory); -void sql_close_database(void); int configure_sqlite_database(sqlite3 *database, int target_version, const char *description); // Helpers int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null); int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement); int execute_insert(sqlite3_stmt *res); -int exec_statement_with_uuid(const char *sql, uuid_t *uuid); int db_execute(sqlite3 *database, const char *cmd); -void initialize_thread_key_pool(void); - -// Look up functions -int get_node_id(uuid_t *host_id, uuid_t *node_id); -struct node_instance_list *get_node_list(void); -void sql_load_node_id(RRDHOST *host); +char *get_database_extented_error(sqlite3 *database, int i, const char *description); -// Help build archived hosts in memory when agent starts -void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info); -RRDLABELS *sql_load_host_labels(uuid_t *host_id); +void sql_drop_table(const char *table); +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv); -// TODO: move to metadata -int update_node_id(uuid_t *host_id, uuid_t *node_id); +uint64_t sqlite_get_db_space(sqlite3 *db); -void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id); +int get_free_page_count(sqlite3 *database); +int get_database_page_count(sqlite3 *database); -// Provide statistics -int sql_metadata_cache_stats(int op); +int sqlite_library_init(void); +void sqlite_library_shutdown(void); -void sql_drop_table(const char *table); -void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv); +void sql_close_database(sqlite3 *database, const char *database_name); +void sqlite_close_databases(void); #endif //NETDATA_SQLITE_FUNCTIONS_H diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c index d074e06d79d4ef..6dcb8f86f8a1c4 100644 --- a/src/database/sqlite/sqlite_metadata.c +++ b/src/database/sqlite/sqlite_metadata.c @@ -1,6 +1,91 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "sqlite_metadata.h" +#include "sqlite3recover.h" +//#include "sqlite_db_migration.h" + +#define DB_METADATA_VERSION 16 + +const char *database_config[] = { + "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, " + "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, " + "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default ''," + "hops INT NOT NULL DEFAULT 0," + "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0," + "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', " + "entries INT NOT NULL DEFAULT 0," + "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)", + + "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, " + "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, " + "chart_type int, memory_mode int, history_entries)", + + "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, " + "multiplier int, divisor int , algorithm int, options text)", + + "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)", + + "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " + "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))", + + "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)", + + "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, " + "on_key text, class text, component text, type text, os text, hosts text, lookup text, " + "every text, units text, calc text, families text, plugin text, module text, charts text, green text, " + "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, " + "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, " + "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text)", + + "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, " + "date_created INT, PRIMARY KEY(host_id, system_key))", + + "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, " + "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))", + + "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)" + " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END", + + "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, " + "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, " + "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))", + + "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, " + "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, " + "flags int, exec_run_timestamp int, delay_up_to_timestamp int, " + "info text, exec_code int, new_status real, old_status real, delay int, " + "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)", + + "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)", + "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)", + "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)", + + NULL +}; + +const char *database_cleanup[] = { + "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)", + "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)", + "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)", + "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)", + "DROP TRIGGER IF EXISTS tr_dim_del", + "DROP INDEX IF EXISTS ind_d1", + "DROP INDEX IF EXISTS ind_c1", + "DROP INDEX IF EXISTS ind_c2", + "DROP INDEX IF EXISTS alert_hash_index", + "DROP INDEX IF EXISTS health_log_d_ind_4", + "DROP INDEX IF EXISTS health_log_d_ind_1", + "DROP INDEX IF EXISTS health_log_d_ind_5", + NULL +}; + +sqlite3 *db_meta = NULL; // SQL statements @@ -131,6 +216,568 @@ struct thread_unittest { unsigned *done; }; +int sql_metadata_cache_stats(int op) +{ + int count, dummy; + + if (unlikely(!db_meta)) + return 0; + + netdata_thread_disable_cancelability(); + sqlite3_db_status(db_meta, op, &count, &dummy, 0); + netdata_thread_enable_cancelability(); + return count; +} + +static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) +{ + if (unlikely(!host)) + return; + + if (unlikely(!node_id)) { + freez(host->node_id); + __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED); + return; + } + + struct aclk_sync_cfg_t *wc = host->aclk_config; + + if (unlikely(!host->node_id)) { + uuid_t *t = mallocz(sizeof(*host->node_id)); + uuid_copy(*t, *node_id); + __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED); + } + else { + uuid_copy(*(host->node_id), *node_id); + } + + if (unlikely(!wc)) + sql_create_aclk_table(host, &host->host_uuid, node_id); + else + uuid_unparse_lower(*node_id, wc->node_id); +} + +#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id" + +int update_node_id(uuid_t *host_id, uuid_t *node_id) +{ + sqlite3_stmt *res = NULL; + RRDHOST *host = NULL; + int rc = 2; + + char host_guid[GUID_LEN + 1]; + uuid_unparse_lower(*host_id, host_guid); + rrd_wrlock(); + host = rrdhost_find_by_guid(host_guid); + if (likely(host)) + set_host_node_id(host, node_id); + rrd_unlock(); + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return 1; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store node instance information"); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to store node instance information"); + goto failed; + } + + rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to store node instance information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store node instance information, rc = %d", rc); + rc = sqlite3_changes(db_meta); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when storing node instance information"); + + return rc - 1; +} + +#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL" + +int get_node_id(uuid_t *host_id, uuid_t *node_id) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return 1; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to select node instance information for a host"); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to select node instance information"); + goto failed; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW && node_id)) + uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0))); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when selecting node instance information"); + + return (rc == SQLITE_ROW) ? 0 : -1; +} + +#define SQL_INVALIDATE_NODE_INSTANCES \ + "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \ + "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))" + +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to invalidate node instance ids"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to invalidate node instance information"); + goto failed; + } + + if (claim_id) + rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, 2); + + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind claim_id parameter to invalidate node instance information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to invalidate node instance information, rc = %d", rc); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when invalidating node instance information"); +} + +#define SQL_GET_NODE_INSTANCE_LIST \ + "SELECT ni.node_id, ni.host_id, h.hostname " \ + "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0" + +struct node_instance_list *get_node_list(void) +{ + struct node_instance_list *node_list = NULL; + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return NULL; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to get node instance information"); + return NULL; + } + + int row = 0; + char host_guid[UUID_STR_LEN]; + while (sqlite3_step_monitored(res) == SQLITE_ROW) + row++; + + if (sqlite3_reset(res) != SQLITE_OK) { + error_report("Failed to reset the prepared statement while fetching node instance information"); + goto failed; + } + node_list = callocz(row + 1, sizeof(*node_list)); + int max_rows = row; + row = 0; + // TODO: Check to remove lock + rrd_rdlock(); + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t)) + uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0))); + if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) { + uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1); + uuid_unparse_lower(*host_id, host_guid); + RRDHOST *host = rrdhost_find_by_guid(host_guid); + if (!host) + continue; + if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) { + netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host)); + continue; + } + uuid_copy(node_list[row].host_id, *host_id); + node_list[row].queryable = 1; + node_list[row].live = (host && (host == localhost || host->receiver + || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0; + node_list[row].hops = (host && host->system_info) ? host->system_info->hops : + uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0; + node_list[row].hostname = + sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL; + } + row++; + if (row == max_rows) + break; + } + rrd_unlock(); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when fetching node instance information"); + + return node_list; +} + +#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id" + +void sql_load_node_id(RRDHOST *host) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch node id"); + return; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to load node instance information"); + goto failed; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW)) { + if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t))) + set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0)); + else + set_host_node_id(host, NULL); + } + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when loading node instance information"); +} + +#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id" + +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info) +{ + int rc; + + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to read host information"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter host information"); + goto skip; + } + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0), + (char *) sqlite3_column_text(res, 1)); + } + +skip: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading host information"); +} + +#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \ + "AND label_key IS NOT NULL AND label_value IS NOT NULL" + +RRDLABELS *sql_load_host_labels(uuid_t *host_id) +{ + int rc; + + RRDLABELS *labels = NULL; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to read host information"); + return NULL; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter host information"); + goto skip; + } + + labels = rrdlabels_create(); + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2)); + } + +skip: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading host information"); + return labels; +} + +static int exec_statement_with_uuid(const char *sql, uuid_t *uuid) +{ + int rc, result = 1; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement %s, rc = %d", sql, rc); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc); + goto skip; + } + + rc = execute_insert(res); + if (likely(rc == SQLITE_DONE)) + result = SQLITE_OK; + else + error_report("Failed to execute %s, rc = %d", sql, rc); + +skip: + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize statement %s, rc = %d", sql, rc); + return result; +} + +static void recover_database(const char *sqlite_database, const char *new_sqlite_database) +{ + sqlite3 *database; + int rc = sqlite3_open(sqlite_database, &database); + if (rc != SQLITE_OK) + return; + + netdata_log_info("Recover %s", sqlite_database); + netdata_log_info(" to %s", new_sqlite_database); + + // This will remove the -shm and -wal files when we close the database + (void) db_execute(database, "select count(*) from sqlite_master limit 0"); + + sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database); + if (recover) { + + rc = sqlite3_recover_run(recover); + + if (rc == SQLITE_OK) + netdata_log_info("Recover complete"); + else + netdata_log_error("Recover encountered an error but the database may be usable"); + + rc = sqlite3_recover_finish(recover); + + (void) sqlite3_close(database); + + if (rc == SQLITE_OK) { + rc = rename(new_sqlite_database, sqlite_database); + if (rc == 0) { + netdata_log_info("Renamed %s", new_sqlite_database); + netdata_log_info(" to %s", sqlite_database); + } + } + else + netdata_log_error("Recover failed to free resources"); + } + else + (void) sqlite3_close(database); +} + + +static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + uuid_t uuid; + + if ( argc != 1 ){ + sqlite3_result_null(context); + return ; + } + int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid); + if (rc == -1) { + sqlite3_result_null(context); + return ; + } + + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); +} + +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + if (argc != 1 ){ + sqlite3_result_null(context); + return ; + } + + if (sqlite3_value_int(argv[0]) != 0) { + struct timespec req = {.tv_sec = 0, .tv_nsec = 1}; + nanosleep(&req, NULL); + } + + sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec()); +} + +void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + (void)argc; + (void)argv; + + uuid_t uuid; + uuid_generate_random(uuid); + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); +} + +// Init +/* + * Initialize the SQLite database + * Return 0 on success + */ +int sql_init_meta_database(db_check_action_type_t rebuild, int memory) +{ + char *err_msg = NULL; + char sqlite_database[FILENAME_MAX + 1]; + int rc; + + if (likely(!memory)) { + snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir); + rc = unlink(sqlite_database); + snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir); + + if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) { + char new_sqlite_database[FILENAME_MAX + 1]; + snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir); + recover_database(sqlite_database, new_sqlite_database); + if (rebuild & DB_CHECK_RECOVER) + return 0; + } + } + else + strncpyz(sqlite_database, ":memory:", sizeof(sqlite_database) - 1); + + rc = sqlite3_open(sqlite_database, &db_meta); + if (rc != SQLITE_OK) { + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); + char *error_str = get_database_extented_error(db_meta, 0, "meta_open"); + if (error_str) + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); + freez(error_str); + sqlite3_close(db_meta); + db_meta = NULL; + return 1; + } + + if (rebuild & DB_CHECK_RECLAIM_SPACE) { + netdata_log_info("Reclaiming space of %s", sqlite_database); + rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg); + if (rc != SQLITE_OK) { + error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg); + sqlite3_free(err_msg); + } + else { + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); + (void) sqlite3_close(db_meta); + } + return 1; + } + + if (rebuild & DB_CHECK_ANALYZE) { + errno = 0; + netdata_log_info("Running ANALYZE on %s", sqlite_database); + rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg); + if (rc != SQLITE_OK) { + error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg); + sqlite3_free(err_msg); + } + else { + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); + (void) sqlite3_close(db_meta); + } + return 1; + } + + netdata_log_info("SQLite database %s initialization", sqlite_database); + + rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal u2h function"); + + rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal now_usec function"); + + rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal uuid_random function"); + + int target_version = DB_METADATA_VERSION; + + if (likely(!memory)) + target_version = perform_database_migration(db_meta, DB_METADATA_VERSION); + + if (configure_sqlite_database(db_meta, target_version, "meta_config")) + return 1; + + if (init_database_batch(db_meta, &database_config[0], "meta_init")) + return 1; + + if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup")) + return 1; + + netdata_log_info("SQLite database initialization completed"); + + return 0; +} // Metadata functions @@ -691,35 +1338,6 @@ static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_st #endif } -int get_pragma_value(sqlite3 *database, const char *sql) -{ - sqlite3_stmt *res = NULL; - int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) - return -1; - - int result = -1; - rc = sqlite3_step_monitored(res); - if (likely(rc == SQLITE_ROW)) - result = sqlite3_column_int(res, 0); - - rc = sqlite3_finalize(res); - (void) rc; - - return result; -} - - -int get_free_page_count(sqlite3 *database) -{ - return get_pragma_value(database, "PRAGMA freelist_count"); -} - -int get_database_page_count(sqlite3 *database) -{ - return get_pragma_value(database, "PRAGMA page_count"); -} - static bool run_cleanup_loop( sqlite3_stmt *res, struct metadata_wc *wc, diff --git a/src/database/sqlite/sqlite_metadata.h b/src/database/sqlite/sqlite_metadata.h index 8441a51b392447..a276f5c4d660f0 100644 --- a/src/database/sqlite/sqlite_metadata.h +++ b/src/database/sqlite/sqlite_metadata.h @@ -6,6 +6,24 @@ #include "sqlite3.h" #include "sqlite_functions.h" +// return a node list +struct node_instance_list { + uuid_t node_id; + uuid_t host_id; + char *hostname; + int live; + int queryable; + int hops; +}; + +typedef enum db_check_action_type { + DB_CHECK_NONE = (1 << 0), + DB_CHECK_RECLAIM_SPACE = (1 << 1), + DB_CHECK_ANALYZE = (1 << 2), + DB_CHECK_CONT = (1 << 3), + DB_CHECK_RECOVER = (1 << 4), +} db_check_action_type_t; + // To initialize and shutdown void metadata_sync_init(void); void metadata_sync_shutdown(void); @@ -20,6 +38,22 @@ void metadata_queue_load_host_context(RRDHOST *host); void metadata_delete_host_chart_labels(char *machine_guid); void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc); +int sql_metadata_cache_stats(int op); + +int get_node_id(uuid_t *host_id, uuid_t *node_id); +int update_node_id(uuid_t *host_id, uuid_t *node_id); +struct node_instance_list *get_node_list(void); +void sql_load_node_id(RRDHOST *host); + +// Help build archived hosts in memory when agent starts +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info); +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id); +RRDLABELS *sql_load_host_labels(uuid_t *host_id); + +uint64_t sqlite_get_meta_space(void); +int sql_init_meta_database(db_check_action_type_t rebuild, int memory); +void sql_close_meta_database(void); + // UNIT TEST int metadata_unittest(void); #endif //NETDATA_SQLITE_METADATA_H diff --git a/src/ml/ml.cc b/src/ml/ml.cc index 057b8820b59f38..33a01f2c6fe29b 100644 --- a/src/ml/ml.cc +++ b/src/ml/ml.cc @@ -1809,13 +1809,16 @@ void ml_init() } } +uint64_t sqlite_get_ml_space(void) +{ + return sqlite_get_db_space(db); +} + void ml_fini() { if (!Cfg.enable_anomaly_detection || !db) return; - int rc = sqlite3_close_v2(db); - if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc)); + sql_close_database(db, "ML"); db = NULL; } diff --git a/src/ml/ml.h b/src/ml/ml.h index f6e7a5436480a6..5fe333bf7df264 100644 --- a/src/ml/ml.h +++ b/src/ml/ml.h @@ -46,6 +46,7 @@ void ml_update_global_statistics_charts(uint64_t models_consulted); bool ml_host_get_host_status(RRDHOST *rh, struct ml_metrics_statistics *mlm); bool ml_host_running(RRDHOST *rh); +uint64_t sqlite_get_ml_space(void); #ifdef __cplusplus };