Reorganize and cleanup database related code (netdata#17101)

* Rearrange code * Explicit init of sqlite library and shutdown Function to close all databases * Add functions to return db size * Prevent overflow * Address review comments * Initialize sqlite library before ml_init * Fix unittest
webfutureiorepo · Mar 5, 2024 · 1116111 · 1116111
1 parent 935ffce
commit 1116111
Show file tree

Hide file tree

Showing 13 changed files with 799 additions and 770 deletions.
diff --git a/src/daemon/main.c b/src/daemon/main.c
@@ -448,11 +448,10 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 #endif
     }
 
-    sql_close_context_database();
-    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
+    sqlite_close_databases();
+    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);
+    sqlite_library_shutdown();
 
-    sql_close_database();
-    watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
 
     // unlink the pid
     if(pidfile[0]) {
@@ -1500,22 +1499,24 @@ int main(int argc, char **argv) {
 #endif
 
                         if(strcmp(optarg, "sqlite-meta-recover") == 0) {
-                            sql_init_database(DB_CHECK_RECOVER, 0);
+                            sql_init_meta_database(DB_CHECK_RECOVER, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "sqlite-compact") == 0) {
-                            sql_init_database(DB_CHECK_RECLAIM_SPACE, 0);
+                            sql_init_meta_database(DB_CHECK_RECLAIM_SPACE, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "sqlite-analyze") == 0) {
-                            sql_init_database(DB_CHECK_ANALYZE, 0);
+                            sql_init_meta_database(DB_CHECK_ANALYZE, 0);
                             return 0;
                         }
 
                         if(strcmp(optarg, "unittest") == 0) {
                             unittest_running = true;
+                            if (sqlite_library_init())
+                                return 1;
 
                             if (pluginsd_parser_unittest()) return 1;
                             if (unit_test_static_threads()) return 1;
@@ -1539,6 +1540,7 @@ int main(int argc, char **argv) {
                             if (ctx_unittest()) return 1;
                             if (uuid_unittest()) return 1;
                             if (dyncfg_unittest()) return 1;
+                            sqlite_library_shutdown();
                             fprintf(stderr, "\n\nALL TESTS PASSED\n\n");
                             return 0;
                         }
@@ -2054,6 +2056,8 @@ int main(int argc, char **argv) {
                 exit(1);
             }
         }
+        if (sqlite_library_init())
+            fatal("Failed to initialize sqlite library");
 
         // --------------------------------------------------------------------
         // Initialize ML configuration

diff --git a/src/daemon/unit_test.c b/src/daemon/unit_test.c
@@ -2546,7 +2546,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
 
     fprintf(stderr, "Initializing localhost with hostname 'dbengine-stress-test'\n");
 
-    (void) sql_init_database(DB_CHECK_NONE, 1);
+    (void)sql_init_meta_database(DB_CHECK_NONE, 1);
     host = dbengine_rrdhost_find_or_create("dbengine-stress-test");
     if (NULL == host)
         return;

diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c
@@ -85,8 +85,7 @@ void *watcher_main(void *arg)
     watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH);
     watcher_wait_for_step(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING);
     watcher_wait_for_step(WATCHER_STEP_ID_STOP_DBENGINE_TIERS);
-    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB);
-    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB);
+    watcher_wait_for_step(WATCHER_STEP_ID_CLOSE_SQL_DATABASES);
     watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_PID_FILE);
     watcher_wait_for_step(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);
     watcher_wait_for_step(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);
@@ -146,10 +145,8 @@ void watcher_thread_start() {
         "wait for dbengine main cache to finish flushing";
     watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg =
         "stop dbengine tiers";
-    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg =
-        "close SQL context db";
-    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg =
-        "close SQL main db";
+    watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_DATABASES].msg =
+        "close SQL databases";
     watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg =
         "remove pid file";
     watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg =

diff --git a/src/daemon/watcher.h b/src/daemon/watcher.h
@@ -27,8 +27,7 @@ typedef enum {
     WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH,
     WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING,
     WATCHER_STEP_ID_STOP_DBENGINE_TIERS,
-    WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB,
-    WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB,
+    WATCHER_STEP_ID_CLOSE_SQL_DATABASES,
     WATCHER_STEP_ID_REMOVE_PID_FILE,
     WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES,
     WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE,

diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c
@@ -1009,7 +1009,7 @@ void dbengine_init(char *hostname) {
 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
     rrdhost_init();
 
-    if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
+    if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
         if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
             set_late_global_environment(system_info);
             fatal("Failed to initialize SQLite");

diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c
@@ -71,24 +71,6 @@ int sql_init_context_database(int memory)
     return 0;
 }
 
-/*
- * Close the sqlite database
- */
-
-void sql_close_context_database(void)
-{
-    int rc;
-    if (unlikely(!db_context_meta))
-        return;
-
-    netdata_log_info("Closing context SQLite database");
-
-    rc = sqlite3_close_v2(db_context_meta);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));
-    db_context_meta = NULL;
-}
-
 //
 // Fetching data
 //
@@ -422,6 +404,12 @@ int sql_context_cache_stats(int op)
     return count;
 }
 
+
+uint64_t sqlite_get_context_space(void)
+{
+    return sqlite_get_db_space(db_context_meta);
+}
+
 //
 // TESTING FUNCTIONS
 //
@@ -456,7 +444,8 @@ int ctx_unittest(void)
     uuid_t host_uuid;
     uuid_generate(host_uuid);
 
-    initialize_thread_key_pool();
+    if (sqlite_library_init())
+        return 1;
 
     int rc = sql_init_context_database(1);
 
@@ -532,7 +521,8 @@ int ctx_unittest(void)
     ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL);
     netdata_log_info("List context end after delete");
 
-    sql_close_context_database();
+    sql_close_database(db_context_meta, "CONTEXT");
+    sqlite_library_shutdown();
 
     return 0;
 }

diff --git a/src/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h
@@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);
 int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data);
 
 int sql_init_context_database(int memory);
+uint64_t sqlite_get_context_space(void);
 void sql_close_context_database(void);
 int ctx_unittest(void);
 #endif //NETDATA_SQLITE_CONTEXT_H