Reduce EBPF memory usage (netdata#19117)

* Add judy array to hold pids

* Add check

* Create entry if not found

* Fix compilation with internal checks

* Use pointer

src/collectors/ebpf.plugin/ebpf.c

@@ -993,8 +993,8 @@ static inline void ebpf_create_apps_for_module(ebpf_module_t *em, struct ebpf_ta
  */
 static void ebpf_create_apps_charts(struct ebpf_target *root)
 {
-    if (unlikely(!ebpf_pids))
-        return;
+//    if (unlikely(!ebpf_pids))
+//        return;
 
     struct ebpf_target *w;
     int newly_added = 0;
@@ -2675,7 +2675,7 @@ static void ebpf_allocate_common_vectors()
 {
     ebpf_judy_pid.pid_table = ebpf_allocate_pid_aral(NETDATA_EBPF_PID_SOCKET_ARAL_TABLE_NAME,
                                                      sizeof(netdata_ebpf_judy_pid_stats_t));
-    ebpf_pids = callocz((size_t)pid_max, sizeof(ebpf_pid_data_t));
+//    ebpf_pids = callocz((size_t)pid_max, sizeof(ebpf_pid_data_t));
     ebpf_aral_init();
 }
 

src/collectors/ebpf.plugin/ebpf_apps.c

@@ -306,7 +306,52 @@ int ebpf_read_apps_groups_conf(struct ebpf_target **agdt, struct ebpf_target **a
 
 #define MAX_CMDLINE 16384
 
-ebpf_pid_data_t *ebpf_pids = NULL;            // to avoid allocations, we pre-allocate the entire pid space.
+Pvoid_t ebpf_pid_judyL = NULL;
+SPINLOCK ebpf_pid_spinlock = NETDATA_SPINLOCK_INITIALIZER;
+
+void ebpf_pid_del(pid_t pid)
+{
+    spinlock_lock(&ebpf_pid_spinlock);
+    (void) JudyLDel(&ebpf_pid_judyL, (Word_t) pid, PJE0);
+    spinlock_unlock(&ebpf_pid_spinlock);
+}
+
+static ebpf_pid_data_t *ebpf_find_pid_data_unsafe(pid_t pid)
+{
+    ebpf_pid_data_t *pid_data = NULL;
+    Pvoid_t *Pvalue = JudyLGet(ebpf_pid_judyL, (Word_t) pid, PJE0);
+    if (Pvalue)
+        pid_data = *Pvalue;
+    return pid_data;
+}
+
+
+ebpf_pid_data_t *ebpf_find_pid_data(pid_t pid)
+{
+    spinlock_lock(&ebpf_pid_spinlock);
+    ebpf_pid_data_t *pid_data = ebpf_find_pid_data_unsafe(pid);
+    spinlock_unlock(&ebpf_pid_spinlock);
+    return pid_data;
+}
+
+ebpf_pid_data_t *ebpf_find_or_create_pid_data(pid_t pid)
+{
+    spinlock_lock(&ebpf_pid_spinlock);
+    ebpf_pid_data_t *pid_data = ebpf_find_pid_data_unsafe(pid);
+    if (!pid_data) {
+        Pvoid_t *Pvalue = JudyLIns(&ebpf_pid_judyL, (Word_t) pid, PJE0);
+        internal_fatal(!Pvalue || Pvalue == PJERR, "EBPF: pid judy array");
+        if (likely(!*Pvalue))
+            *Pvalue = pid_data = callocz(1, sizeof(*pid_data));
+        else
+            pid_data = *Pvalue;
+    }
+    spinlock_unlock(&ebpf_pid_spinlock);
+
+    return pid_data;
+}
+
+//ebpf_pid_data_t *ebpf_pids = NULL;            // to avoid allocations, we pre-allocate the entire pid space.
 ebpf_pid_data_t *ebpf_pids_link_list = NULL;  // global list of all processes running
 
 size_t ebpf_all_pids_count = 0; // the number of processes running read from /proc
@@ -573,8 +618,9 @@ static inline void link_all_processes_to_their_parents(void)
             continue;
         }
 
-        pp = &ebpf_pids[p->ppid];
-        if (likely(pp->pid)) {
+//        pp = &ebpf_pids[p->ppid];
+        pp = ebpf_find_pid_data(p->ppid);
+        if (likely(pp && pp->pid)) {
             p->parent = pp;
             pp->children_count++;
 
@@ -665,10 +711,14 @@ static void apply_apps_groups_targets_inheritance(void)
     }
 
     // init goes always to default target
-    ebpf_pids[INIT_PID].target = apps_groups_default_target;
+    ebpf_pid_data_t *pid_entry = ebpf_find_or_create_pid_data(INIT_PID);
+    pid_entry->target = apps_groups_default_target;
+//    ebpf_pids[INIT_PID].target = apps_groups_default_target;
 
     // pid 0 goes always to default target
-    ebpf_pids[0].target = apps_groups_default_target;
+    pid_entry = ebpf_find_or_create_pid_data(0);
+    pid_entry->target = apps_groups_default_target;
+    //ebpf_pids[0].target = apps_groups_default_target;
 
     // give a default target on all top level processes
     if (unlikely(debug_enabled))
@@ -684,7 +734,9 @@ static void apply_apps_groups_targets_inheritance(void)
             p->sortlist = sortlist++;
     }
 
-    ebpf_pids[1].sortlist = sortlist++;
+    //ebpf_pids[1].sortlist = sortlist++;
+    pid_entry = ebpf_find_or_create_pid_data(1);
+    pid_entry->sortlist = sortlist++;
 
     // give a target to all merged child processes
     found = 1;
@@ -734,7 +786,9 @@ static inline void post_aggregate_targets(struct ebpf_target *root)
  */
 void ebpf_del_pid_entry(pid_t pid)
 {
-    ebpf_pid_data_t *p = &ebpf_pids[pid];
+
+    //ebpf_pid_data_t *p = &ebpf_pids[pid];
+    ebpf_pid_data_t *p = ebpf_find_pid_data(pid);
 
     debug_log("process %d %s exited, deleting it.", pid, p->comm);
 
@@ -746,7 +800,6 @@ void ebpf_del_pid_entry(pid_t pid)
     if (p->prev)
         p->prev->next = p->next;
 
-
     if ((p->thread_collecting & EBPF_PIDS_PROC_FILE) || p->has_proc_file)
         ebpf_all_pids_count--;
 
@@ -768,7 +821,9 @@ void ebpf_del_pid_entry(pid_t pid)
     }
     rw_spinlock_write_unlock(&ebpf_judy_pid.index.rw_spinlock);
 
-    memset(p, 0, sizeof(ebpf_pid_data_t));
+    freez(p);
+    //memset(p, 0, sizeof(ebpf_pid_data_t));
+    ebpf_pid_del(pid);
 }
 
 /**

src/collectors/ebpf.plugin/ebpf_apps.h

@@ -201,7 +201,7 @@ typedef struct __attribute__((packed)) ebpf_pid_data {
 
 } ebpf_pid_data_t;
 
-extern ebpf_pid_data_t *ebpf_pids;
+//extern ebpf_pid_data_t *ebpf_pids;
 extern ebpf_pid_data_t *ebpf_pids_link_list;
 extern size_t ebpf_all_pids_count;
 extern size_t ebpf_hash_table_pids_count;
@@ -303,8 +303,11 @@ static inline void ebpf_process_release_publish(ebpf_publish_process_t *ptr)
     freez(ptr);
 }
 
+ebpf_pid_data_t *ebpf_find_or_create_pid_data(pid_t pid);
+
 static inline ebpf_pid_data_t *ebpf_get_pid_data(uint32_t pid, uint32_t tgid, char *name, uint32_t idx) {
-    ebpf_pid_data_t *ptr = &ebpf_pids[pid];
+//    ebpf_pid_data_t *ptr = &ebpf_pids[pid];
+    ebpf_pid_data_t *ptr = ebpf_find_or_create_pid_data(pid);
     ptr->thread_collecting |= 1<<idx;
     // The caller is getting data to work.
     if (!name && idx != EBPF_PIDS_PROC_FILE)