Abort on fatal and report system available bytes on allocation failur…

…es. (netdata#19332)

* Abort on non-zero rc when we have sentry.

* Abort on recursive-fatals when sentry is enabled

* Add function that returns last reported system memory.

* Provide available memory in fatal message.

* Update system_memory.c

---------

Co-authored-by: Costa Tsaousis <[email protected]>

src/daemon/daemon-shutdown.c

@@ -296,8 +296,13 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re
 #endif
 
 #ifdef ENABLE_SENTRY
-    nd_sentry_fini();
-#endif
-
+    if (ret) {
+        abort();
+    } else {
+        nd_sentry_fini();
+        exit(ret);
+    }
+#else
     exit(ret);
+#endif
 }

src/libnetdata/libnetdata.c

@@ -410,13 +410,19 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {
 
 char *strdupz(const char *s) {
     char *t = strdup(s);
-    if (unlikely(!t)) fatal("Cannot strdup() string '%s'", s);
+    if (unlikely(!t)) {
+        OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
+        fatal("Cannot strdup() string '%s' (system memory available bytes: %lu)", s, sm.ram_available_bytes);
+    }
     return t;
 }
 
 char *strndupz(const char *s, size_t len) {
     char *t = strndup(s, len);
-    if (unlikely(!t)) fatal("Cannot strndup() string '%s' of len %zu", s, len);
+    if (unlikely(!t)) {
+        OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
+        fatal("Cannot strndup() string '%s' of len %zu (system memory available bytes: %lu)", s, len, sm.ram_available_bytes);
+    }
     return t;
 }
 
@@ -427,19 +433,28 @@ void freez(void *ptr) {
 
 void *mallocz(size_t size) {
     void *p = malloc(size);
-    if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", size);
+    if (unlikely(!p)) {
+        OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
+        fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", size, sm.ram_available_bytes);
+    }
     return p;
 }
 
 void *callocz(size_t nmemb, size_t size) {
     void *p = calloc(nmemb, size);
-    if (unlikely(!p)) fatal("Cannot allocate %zu bytes of memory.", nmemb * size);
+    if (unlikely(!p)) {
+        OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
+        fatal("Cannot allocate %zu bytes of memory (system memory available bytes: %lu)", nmemb * size, sm.ram_available_bytes);
+    }
     return p;
 }
 
 void *reallocz(void *ptr, size_t size) {
     void *p = realloc(ptr, size);
-    if (unlikely(!p)) fatal("Cannot re-allocate memory to %zu bytes.", size);
+    if (unlikely(!p)) {
+        OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
+        fatal("Cannot re-allocate memory to %zu bytes. (system memory available bytes: %lu)", size, sm.ram_available_bytes);
+    }
     return p;
 }
 

src/libnetdata/log/nd_log.c

@@ -419,7 +419,12 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned
         fprintf(stderr, "\nRECURSIVE FATAL STATEMENTS, latest from %s() of %lu@%s, EXITING NOW! 23e93dfccbf64e11aac858b9410d8a82\n",
                 function, line, file);
         fflush(stderr);
+
+#ifdef ENABLE_SENTRY
+        abort();
+#else
         _exit(1);
+#endif
     }
 
     int saved_errno = errno;

src/libnetdata/os/system_memory.c

@@ -2,6 +2,14 @@
 
 #include "libnetdata/libnetdata.h"
 
+static OS_SYSTEM_MEMORY os_system_memory_last = {
+    0, 0,
+};
+
+OS_SYSTEM_MEMORY os_last_reported_system_memory(void) {
+    return os_system_memory_last;
+}
+
 // Windows
 #if defined(OS_WINDOWS)
 #include <windows.h>
@@ -14,6 +22,7 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {
     if (GlobalMemoryStatusEx(&statex)) {
         sm.ram_total_bytes = statex.ullTotalPhys;
         sm.ram_available_bytes = statex.ullAvailPhys;
+        os_system_memory_last = sm;
     }
 
     return sm;
@@ -56,10 +65,11 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {
         mach_port_deallocate(mach_task_self(), mach_port);
     }
 
-    return (OS_SYSTEM_MEMORY){
+    os_system_memory_last = (OS_SYSTEM_MEMORY){
         .ram_total_bytes = total_ram,
         .ram_available_bytes = ram_available,
     };
+    return os_system_memory_last;
 }
 #endif
 
@@ -119,6 +129,7 @@ static OS_SYSTEM_MEMORY os_system_memory_cgroup_v1(bool query_total_ram __maybe_
 
 done:
     sm.ram_available_bytes = sm.ram_total_bytes - (used - inactive);
+    os_system_memory_last = sm;
     return sm;
 
 failed:
@@ -184,6 +195,7 @@ static OS_SYSTEM_MEMORY os_system_memory_cgroup_v2(bool query_total_ram __maybe_
 
 done:
     sm.ram_available_bytes = sm.ram_total_bytes - (used - inactive);
+    os_system_memory_last = sm;
     return sm;
 
 failed:
@@ -224,6 +236,7 @@ OS_SYSTEM_MEMORY os_system_memory_meminfo(bool query_total_ram __maybe_unused) {
     }
 
     // we keep ff open to speed up the next calls
+    os_system_memory_last = sm;
     return sm;
 
 failed:
@@ -342,6 +355,7 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {
 
     sm.ram_available_bytes = (free_pages + inactive_pages) * page_size;
 
+    os_system_memory_last = sm;
     return sm;
 
 failed:

src/libnetdata/os/system_memory.h

@@ -23,4 +23,7 @@ typedef struct {
 // The function to get current system memory:
 OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram);
 
+// Returns the last successfully reported os_system_memory() value.
+OS_SYSTEM_MEMORY os_last_reported_system_memory(void);
+
 #endif //NETDATA_OS_MEM_AVAILABLE_H