[pull] master from netdata:master

.github/scripts/gen-docker-tags.py

@@ -31,4 +31,6 @@
             )
         ])
 
+        tags = tags + tuple([f'{r}:stable' for r in REPOS])
+
 print(','.join(tags))

.github/workflows/docker.yml

@@ -333,7 +333,7 @@ jobs:
       - name: Create and Push Manifest
         id: manifest
         if: github.repository == 'netdata/netdata'
-        run: docker buildx imagetools create "$(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' "${{ needs.gen-tags.outputs.tags }}")"
+        run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' "${{ needs.gen-tags.outputs.tags }}")
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -489,7 +489,7 @@ jobs:
       - name: Create and Push Manifest
         id: manifest
         if: github.repository == 'netdata/netdata'
-        run: docker buildx imagetools create "$(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' "${{ needs.gen-tags.outputs.tags }}")"
+        run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' "${{ needs.gen-tags.outputs.tags }}")
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -645,7 +645,7 @@ jobs:
       - name: Create and Push Manifest
         id: manifest
         if: github.repository == 'netdata/netdata'
-        run: docker buildx imagetools create "$(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' "${{ needs.gen-tags.outputs.tags }}")"
+        run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' "${{ needs.gen-tags.outputs.tags }}")
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -690,7 +690,7 @@ jobs:
           repo: netdata/helmchart
           workflow: Agent Version PR
           ref: refs/heads/master
-          inputs: '{"agent_version": "${{ inputs.version }}"}'
+          inputs: '{"agent_version": "v${{ inputs.version }}"}'
       - name: Trigger MSI build
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata'
         id: trigger-msi

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 **Merged pull requests:**
 
+- set min thread stack size to 1 MB [\#17317](https://github.com/netdata/netdata/pull/17317) ([ilyam8](https://github.com/ilyam8))
+- Drop legacy dbengine support [\#17315](https://github.com/netdata/netdata/pull/17315) ([stelfrag](https://github.com/stelfrag))
+- Fix assorted issues in the Docker build process. [\#17312](https://github.com/netdata/netdata/pull/17312) ([Ferroin](https://github.com/Ferroin))
+- dyncfg function on parents should not require any access rights [\#17310](https://github.com/netdata/netdata/pull/17310) ([ktsaou](https://github.com/ktsaou))
 - Update metadata frequency [\#17307](https://github.com/netdata/netdata/pull/17307) ([stelfrag](https://github.com/stelfrag))
 - Fix handling of post-release workflows triggered by Docker workflow. [\#17306](https://github.com/netdata/netdata/pull/17306) ([Ferroin](https://github.com/Ferroin))
 - go.d: sd ll: add mysql socket jobs [\#17305](https://github.com/netdata/netdata/pull/17305) ([ilyam8](https://github.com/ilyam8))
@@ -402,7 +406,6 @@
 - Add the Mobile App notification Integration [\#16715](https://github.com/netdata/netdata/pull/16715) ([sashwathn](https://github.com/sashwathn))
 - Update GHA steps that handle artifacts to use latest versions of upload/download actions. [\#16714](https://github.com/netdata/netdata/pull/16714) ([Ferroin](https://github.com/Ferroin))
 - CI runtime check cleanup [\#16713](https://github.com/netdata/netdata/pull/16713) ([Ferroin](https://github.com/Ferroin))
-- disable logsmanagement when installing on macOS [\#16708](https://github.com/netdata/netdata/pull/16708) ([ilyam8](https://github.com/ilyam8))
 
 ## [v1.44.3](https://github.com/netdata/netdata/tree/v1.44.3) (2024-02-12)
 

packaging/version

@@ -1 +1 @@
-v1.45.0-100-nightly
+v1.45.0-106-nightly

src/daemon/config/dyncfg.c

@@ -438,7 +438,7 @@ void dyncfg_add_streaming(BUFFER *wb) {
                    , 120
                    , "Dynamic configuration"
                    , "config"
-                   , (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA)
+                   , HTTP_ACCESS_ANONYMOUS_DATA
                    , 1000
     );
 }

src/daemon/global_statistics.c

@@ -2562,12 +2562,10 @@ static void dbengine2_statistics_charts(void) {
                     if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue;
                     if(!host->db[tier].si) continue;
 
-                    if(is_storage_engine_shared(host->db[tier].si)) {
-                        if(counted_multihost_db[tier])
-                            continue;
-                        else
-                            counted_multihost_db[tier] = 1;
-                    }
+                    if(counted_multihost_db[tier])
+                        continue;
+                    else
+                        counted_multihost_db[tier] = 1;
 
                     ++dbengine_contexts;
                     rrdeng_get_37_statistics((struct rrdengine_instance *)host->db[tier].si, local_stats_array);

src/daemon/main.c

@@ -2023,6 +2023,9 @@ int main(int argc, char **argv) {
 
         // setup threads configs
         default_stacksize = netdata_threads_init();
+        // musl default thread stack size is 128k, let's set it to a higher value to avoid random crashes
+        if (default_stacksize < 1 * 1024 * 1024)
+            default_stacksize = 1 * 1024 * 1024;
 
 #ifdef NETDATA_INTERNAL_CHECKS
         config_set_boolean(CONFIG_SECTION_PLUGINS, "netdata monitoring", true);

src/database/engine/rrdengine.c

@@ -1366,8 +1366,7 @@ static void *ctx_shutdown_tp_worker(struct rrdengine_instance *ctx __maybe_unuse
         if(!logged) {
             logged = true;
             netdata_log_info("DBENGINE: waiting for %zu inflight queries to finish to shutdown tier %d...",
-                 __atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED),
-                 (ctx->config.legacy) ? -1 : ctx->config.tier);
+                 __atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED), ctx->config.tier);
         }
         sleep_usec(1 * USEC_PER_MS);
     }

src/database/engine/rrdengine.h

@@ -350,8 +350,6 @@ extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of dele
 
 struct rrdengine_instance {
     struct {
-        bool legacy;                                // true when the db is autonomous for a single host
-
         int tier;                                   // the tier of this ctx
         uint8_t page_type;                          // default page type for this context
 

src/database/engine/rrdengineapi.c

@@ -98,26 +98,18 @@ void rrdeng_metrics_group_release(STORAGE_INSTANCE *si __maybe_unused, STORAGE_M
 // metric handle for legacy dbs
 
 /* This UUID is not unique across hosts */
-void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
+void rrdeng_generate_unittest_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid)
 {
-    EVP_MD_CTX *evpctx;
-    unsigned char hash_value[EVP_MAX_MD_SIZE];
-    unsigned int hash_len;
-
-    evpctx = EVP_MD_CTX_create();
-    EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
-    EVP_DigestUpdate(evpctx, dim_id, strlen(dim_id));
-    EVP_DigestUpdate(evpctx, chart_id, strlen(chart_id));
-    EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
-    EVP_MD_CTX_destroy(evpctx);
-    fatal_assert(hash_len > sizeof(uuid_t));
-    memcpy(ret_uuid, hash_value, sizeof(uuid_t));
+    CLEAN_BUFFER *wb = buffer_create(100, NULL);
+    buffer_sprintf(wb,"%s.%s", dim_id, chart_id);
+    UUID uuid = UUID_generate_from_hash(buffer_tostring(wb), buffer_strlen(wb));
+    uuid_copy(*ret_uuid, uuid.uuid);
 }
 
-static METRIC *rrdeng_metric_get_legacy(STORAGE_INSTANCE *si, const char *rd_id, const char *st_id) {
+static METRIC *rrdeng_metric_unittest(STORAGE_INSTANCE *si, const char *rd_id, const char *st_id) {
     struct rrdengine_instance *ctx = (struct rrdengine_instance *)si;
     uuid_t legacy_uuid;
-    rrdeng_generate_legacy_uuid(rd_id, st_id, &legacy_uuid);
+    rrdeng_generate_unittest_uuid(rd_id, st_id, &legacy_uuid);
     return mrg_metric_get_and_acquire(main_mrg, &legacy_uuid, (Word_t) ctx);
 }
 
@@ -165,11 +157,8 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE
     metric = mrg_metric_get_and_acquire(main_mrg, &rd->metric_uuid, (Word_t) ctx);
 
     if(unlikely(!metric)) {
-        if(unlikely(ctx->config.legacy)) {
-            // this is a single host database
-            // generate uuid from the chart and dimensions ids
-            // and overwrite the one supplied by rrddim
-            metric = rrdeng_metric_get_legacy(si, rrddim_id(rd), rrdset_id(rd->rrdset));
+        if(unlikely(unittest_running)) {
+            metric = rrdeng_metric_unittest(si, rrddim_id(rd), rrdset_id(rd->rrdset));
             if (metric)
                 uuid_copy(rd->metric_uuid, *mrg_metric_uuid(main_mrg, metric));
         }
@@ -1128,11 +1117,6 @@ void rrdeng_readiness_wait(struct rrdengine_instance *ctx) {
     netdata_log_info("DBENGINE: tier %d is ready for data collection and queries", ctx->config.tier);
 }
 
-bool rrdeng_is_legacy(STORAGE_INSTANCE *si) {
-    struct rrdengine_instance *ctx = (struct rrdengine_instance *)si;
-    return ctx->config.legacy;
-}
-
 void rrdeng_exit_mode(struct rrdengine_instance *ctx) {
     __atomic_store_n(&ctx->quiesce.exit_mode, true, __ATOMIC_RELAXED);
 }
@@ -1159,14 +1143,11 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
         return UV_EMFILE;
     }
 
-    if(NULL == ctxp) {
+    if(ctxp)
+        *ctxp = ctx = callocz(1, sizeof(*ctx));
+    else {
         ctx = multidb_ctx[tier];
         memset(ctx, 0, sizeof(*ctx));
-        ctx->config.legacy = false;
-    }
-    else {
-        *ctxp = ctx = callocz(1, sizeof(*ctx));
-        ctx->config.legacy = true;
     }
 
     ctx->config.tier = (int)tier;
@@ -1192,7 +1173,7 @@ int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
         return 0;
     }
 
-    if (ctx->config.legacy) {
+    if (unittest_running) {
         freez(ctx);
         if (ctxp)
             *ctxp = NULL;
@@ -1223,17 +1204,17 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
     size_t count = 10;
     while(__atomic_load_n(&ctx->atomic.collectors_running, __ATOMIC_RELAXED) && count && !unittest_running) {
         if(!logged) {
-            netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", (ctx->config.legacy) ? -1 : ctx->config.tier);
+            netdata_log_info("DBENGINE: waiting for collectors to finish on tier %d...", ctx->config.tier);
             logged = true;
         }
         sleep_usec(100 * USEC_PER_MS);
         count--;
     }
 
-    netdata_log_info("DBENGINE: flushing main cache for tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier);
+    netdata_log_info("DBENGINE: flushing main cache for tier %d", ctx->config.tier);
     pgc_flush_all_hot_and_dirty_pages(main_cache, (Word_t)ctx);
 
-    netdata_log_info("DBENGINE: shutting down tier %d", (ctx->config.legacy) ? -1 : ctx->config.tier);
+    netdata_log_info("DBENGINE: shutting down tier %d", ctx->config.tier);
     struct completion completion = {};
     completion_init(&completion);
     rrdeng_enq_cmd(ctx, RRDENG_OPCODE_CTX_SHUTDOWN, NULL, &completion, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL);
@@ -1242,7 +1223,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx) {
 
     finalize_rrd_files(ctx);
 
-    if(ctx->config.legacy)
+    if (unittest_running) //(ctx->config.unittest)
         freez(ctx);
 
     rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);

src/database/engine/rrdengineapi.h

@@ -24,8 +24,6 @@ extern uint8_t tier_page_type[];
 
 #define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]
 
-void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
-
 STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si);
 STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *si, uuid_t *uuid);
 void rrdeng_metric_release(STORAGE_METRIC_HANDLE *smh);
@@ -221,6 +219,5 @@ struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void);
 
 RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
 size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
-bool rrdeng_is_legacy(STORAGE_INSTANCE *si);
 
 #endif /* NETDATA_RRDENGINEAPI_H */

src/database/engine/rrdenginelib.c

@@ -76,51 +76,6 @@ int open_file_for_io(char *path, int flags, uv_file *file, int direct)
     return fd;
 }
 
-int is_legacy_child(const char *machine_guid)
-{
-    uuid_t uuid;
-    char  dbengine_file[FILENAME_MAX+1];
-
-    if (unlikely(!strcmp(machine_guid, "unittest-dbengine") || !strcmp(machine_guid, "dbengine-dataset") ||
-                 !strcmp(machine_guid, "dbengine-stress-test"))) {
-        return 1;
-    }
-    if (!uuid_parse(machine_guid, uuid)) {
-        uv_fs_t stat_req;
-        snprintfz(dbengine_file, FILENAME_MAX, "%s/%s/dbengine", netdata_configured_cache_dir, machine_guid);
-        int rc = uv_fs_stat(NULL, &stat_req, dbengine_file, NULL);
-        if (likely(rc == 0 && ((stat_req.statbuf.st_mode & S_IFMT) == S_IFDIR))) {
-            //netdata_log_info("Found legacy engine folder \"%s\"", dbengine_file);
-            return 1;
-        }
-    }
-    return 0;
-}
-
-int count_legacy_children(char *dbfiles_path)
-{
-    int ret;
-    uv_fs_t req;
-    uv_dirent_t dent;
-    int legacy_engines = 0;
-
-    ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
-    if (ret < 0) {
-        uv_fs_req_cleanup(&req);
-        netdata_log_error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
-        return ret;
-    }
-
-    while(UV_EOF != uv_fs_scandir_next(&req, &dent)) {
-        if (dent.type == UV_DIRENT_DIR) {
-            if (is_legacy_child(dent.name))
-                legacy_engines++;
-        }
-    }
-    uv_fs_req_cleanup(&req);
-    return legacy_engines;
-}
-
 int compute_multidb_diskspace()
 {
     char multidb_disk_space_file[FILENAME_MAX + 1];
@@ -139,23 +94,8 @@ int compute_multidb_diskspace()
         }
     }
 
-    if (computed_multidb_disk_quota_mb == -1) {
-        int rc = count_legacy_children(netdata_configured_cache_dir);
-        if (likely(rc >= 0)) {
-            computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
-            netdata_log_info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
-
-            fp = fopen(multidb_disk_space_file, "w");
-            if (likely(fp)) {
-                fprintf(fp, "%d", computed_multidb_disk_quota_mb);
-                netdata_log_info("Created file '%s' to store the computed value", multidb_disk_space_file);
-                fclose(fp);
-            } else
-                netdata_log_error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
-        }
-        else
-            computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
-    }
+    if (computed_multidb_disk_quota_mb == -1)
+        computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
 
     return computed_multidb_disk_quota_mb;
 }

src/database/engine/rrdenginelib.h

@@ -89,6 +89,5 @@ static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
     return open_file_for_io(path, flags, file, 0);
 }
 int compute_multidb_diskspace();
-int is_legacy_child(const char *machine_guid);
 
 #endif /* NETDATA_RRDENGINELIB_H */

src/database/rrd.h

@@ -1369,7 +1369,6 @@ extern netdata_rwlock_t rrd_rwlock;
 
 // ----------------------------------------------------------------------------
 
-bool is_storage_engine_shared(STORAGE_INSTANCE *si);
 void rrdset_index_init(RRDHOST *host);
 void rrdset_index_destroy(RRDHOST *host);