Merge branch 'master' into master

Signed-off-by: AdheipSingh <[email protected]>

CODEOWNERS

@@ -91,6 +91,8 @@
 # Devcontainer
 /.devcontainer                  @patrick-stephens @niedbalski @edsiper
 
-# Calytia Fleet
-/plugins/custom_calyptia/       @pwhelan
-/plugins/in_calyptia_fleet/     @pwhelan
+# Calyptia Fleet
+/include/fluent-bit/calyptia/       @pwhelan @patrick-stephens @niedbalski
+/plugins/custom_calyptia/       @pwhelan @patrick-stephens @niedbalski
+/plugins/custom_calyptia/       @pwhelan @patrick-stephens @niedbalski
+/plugins/out_calyptia/     @pwhelan @patrick-stephens @niedbalski

Vagrantfile

@@ -28,7 +28,7 @@ Vagrant.configure("2") do |config|
     # Main build
     apt-get install --yes build-essential cmake dh-make git make openssl pkg-config tar
     # Dependencies
-    apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5
+    apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5 libbpf-dev
 
     # Debug
     apt-get install --yes gdb valgrind

include/CMakeLists.txt

@@ -10,6 +10,12 @@ install(FILES ${headers}
     COMPONENT headers
     PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ)
 
+file(GLOB headers "fluent-bit/calyptia/*.h")
+install(FILES ${headers}
+    DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit
+    COMPONENT headers
+    PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ)
+
 file(GLOB headers "fluent-bit/config_format/*.h")
 install(FILES ${headers}
     DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/config_format/

include/fluent-bit/calyptia/calyptia_constants.h

@@ -0,0 +1,68 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2015-2024 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+#ifndef FLB_CALYPTIA_CONSTANTS_H
+#define FLB_CALYPTIA_CONSTANTS_H
+
+/* End point */
+#define DEFAULT_CALYPTIA_HOST     "cloud-api.calyptia.com"
+#define DEFAULT_CALYPTIA_PORT     "443"
+
+/* HTTP action types */
+#define CALYPTIA_ACTION_REGISTER  0
+#define CALYPTIA_ACTION_PATCH     1
+#define CALYPTIA_ACTION_METRICS   2
+#define CALYPTIA_ACTION_TRACE     3
+
+/* Endpoints */
+#define CALYPTIA_ENDPOINT_CREATE  "/v1/agents"
+#define CALYPTIA_ENDPOINT_PATCH   "/v1/agents/%s"
+#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics"
+#define CALYPTIA_ENDPOINT_TRACE   "/v1/traces/%s"
+
+#define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets"
+#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini"
+#define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files"
+
+/* Storage */
+#define CALYPTIA_SESSION_FILE     "session.CALYPTIA"
+
+/* Headers */
+#define CALYPTIA_HEADERS_PROJECT       "X-Project-Token"
+#define CALYPTIA_HEADERS_AGENT_TOKEN   "X-Agent-Token"
+#define CALYPTIA_HEADERS_CTYPE         "Content-Type"
+#define CALYPTIA_HEADERS_CTYPE_JSON    "application/json"
+#define CALYPTIA_HEADERS_CTYPE_MSGPACK "application/x-msgpack"
+
+#ifndef FLB_SYSTEM_WINDOWS
+#define FLEET_DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet"
+#else
+#define FLEET_DEFAULT_CONFIG_DIR NULL
+#endif
+
+#ifndef PATH_SEPARATOR
+#ifndef FLB_SYSTEM_WINDOWS
+#define PATH_SEPARATOR "/"
+#else
+#define PATH_SEPARATOR "\\"
+#endif
+#endif /* PATH_SEPARATOR */
+
+#define CALYPTIA_MAX_DIR_SIZE 4096
+
+#endif /* FLB_CALYPTIA_CONSTANTS_H */

include/fluent-bit/flb_utils.h

@@ -74,5 +74,6 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size);
 void flb_utils_set_plugin_string_property(const char *name,
                                           flb_sds_t *field_storage,
                                           flb_sds_t  new_value);
+int flb_utils_mkdir(const char *dir, int perms);
 
 #endif

lib/cprofiles/include/cprofiles/cprof_info.h

@@ -20,7 +20,7 @@
 #ifndef CPROF_INFO_H
 #define CPROF_INFO_H
 
-#define CPROF_SOURCE_DIR "/Users/adheipsingh/parseable/fluent-bit"
+#define CPROF_SOURCE_DIR "/src/fluent-bit"
 
 /* General flags set by /CMakeLists.txt */
 #ifndef CPROF_HAVE_TIMESPEC_GET

plugins/custom_calyptia/calyptia.c

@@ -27,11 +27,25 @@
 #include <fluent-bit/flb_input.h>
 #include <fluent-bit/flb_filter.h>
 #include <fluent-bit/flb_output.h>
-
+#include <fluent-bit/flb_custom_plugin.h>
+#include <fluent-bit/flb_config.h>
+#include <fluent-bit/flb_config_map.h>
+#include <fluent-bit/flb_utils.h>
 #include <fluent-bit/flb_hash.h>
 
+#include <fluent-bit/calyptia/calyptia_constants.h>
+
 #include "calyptia.h"
 
+#define UUID_BUFFER_SIZE 38 /* Maximum length of UUID string + null terminator */
+
+/* Function wrappers to enable mocking for unit test filesystem access */
+int (*flb_access)(const char *pathname, int mode) = access;
+int (*flb_open)(const char *pathname, int flags, ...) = open;
+ssize_t (*flb_write)(int fd, const void *buf, size_t count) = write;
+int (*flb_close)(int fd) = close;
+int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size) = flb_utils_read_file;
+
 /*
  * Check if the key belongs to a sensitive data field, if so report it. We never
  * share any sensitive data.
@@ -217,16 +231,13 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *
     flb_input_set_property(fleet, "api_key", ctx->api_key);
     flb_input_set_property(fleet, "host", ctx->cloud_host);
     flb_input_set_property(fleet, "port", ctx->cloud_port);
+    flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir);
 
     /* Set TLS properties */
     flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off");
     flb_input_set_property(fleet, "tls.verify", ctx->cloud_tls_verify == 1 ? "on" : "off");
 
     /* Optional configurations */
-    if (ctx->fleet_config_dir) {
-        flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir);
-    }
-
     if (ctx->fleet_max_http_buffer_size) {
         flb_input_set_property(fleet, "max_http_buffer_size", ctx->fleet_max_http_buffer_size);
     }
@@ -376,15 +387,182 @@ static flb_sds_t sha256_to_hex(unsigned char *sha256)
     return hex;
 }
 
-static flb_sds_t get_machine_id(struct calyptia *ctx)
+static flb_sds_t generate_base_agent_directory(struct calyptia *ctx, flb_sds_t *fleet_dir)
 {
-    int ret;
-    char *buf;
-    size_t blen;
+    flb_sds_t ret = NULL;
+
+    if (ctx == NULL || fleet_dir == NULL) {
+        return NULL;
+    }
+
+    if (*fleet_dir == NULL) {
+        *fleet_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE);
+        if (*fleet_dir == NULL) {
+            return NULL;
+        }
+    }
+
+    ret = flb_sds_printf(fleet_dir, "%s", ctx->fleet_config_dir);
+    if (ret == NULL) {
+        flb_sds_destroy(*fleet_dir);
+        return NULL;
+    }
+
+    return ret;
+}
+
+flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname)
+{
+    flb_sds_t cfgname = NULL;
+    flb_sds_t ret;
+
+    if (ctx == NULL || fname == NULL) {
+        return NULL;
+    }
+
+    if (generate_base_agent_directory(ctx, &cfgname) == NULL) {
+        return NULL;
+    }
+
+    ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname);
+    if (ret == NULL) {
+        flb_sds_destroy(cfgname);
+        return NULL;
+    }
+
+    return cfgname;
+}
+
+static char* generate_uuid() {
+    char* uuid = flb_malloc(UUID_BUFFER_SIZE);
+    if (!uuid) {
+        flb_errno();
+        return NULL;
+    }
+
+    /* create new UUID for fleet */
+    if (flb_utils_uuid_v4_gen(uuid) != 0 || strlen(uuid) == 0) {
+        flb_free(uuid);
+        return NULL;
+    }
+    return uuid;
+}
+
+static int write_uuid_to_file(flb_sds_t fleet_machine_id, char* uuid) {
+    int fd;
+    size_t uuid_len;
+
+    if (fleet_machine_id == NULL || uuid == NULL) {
+        return FLB_FALSE;
+    }
+
+    /* write uuid to file */
+    fd = flb_open(fleet_machine_id, O_CREAT | O_WRONLY | O_TRUNC, 0666);
+    if (fd == -1) {
+        return FLB_FALSE;
+    }
+
+    uuid_len = strlen(uuid);
+
+    if (flb_write(fd, uuid, uuid_len) != uuid_len) {
+        flb_close(fd);
+        return FLB_FALSE;
+    }
+
+    flb_close(fd);
+    return FLB_TRUE;
+}
+
+static int create_agent_directory(struct calyptia *ctx)
+{
+    if( ctx == NULL ) {
+        return -1;
+    }
+
+    /* If it exists just return */
+    if (access(ctx->fleet_config_dir, F_OK) == 0) {
+        return 0;
+    }
+
+    /* Create the directory if it does not exist */
+    if (flb_utils_mkdir(ctx->fleet_config_dir, 0700) != 0) {
+        flb_plg_error(ctx->ins, "failed to create directory: %s", ctx->fleet_config_dir);
+        return -1;
+    }
+
+    return 0;
+}
+
+flb_sds_t get_machine_id(struct calyptia *ctx)
+{
+    int ret = -1;
+    char *buf = NULL;
+    size_t blen = 0;
     unsigned char sha256_buf[64] = {0};
 
+#if defined(FLB_SYSTEM_WINDOWS)
     /* retrieve raw machine id */
     ret = flb_utils_get_machine_id(&buf, &blen);
+#else
+    /* /etc/machine-id is not guaranteed to be unique so we generate one */
+    flb_sds_t fleet_machine_id = NULL;
+
+    /** ensure we have the directory created */
+    if (create_agent_directory(ctx) != 0) {
+        return NULL;
+    }
+
+    /** now get the agent filename */
+    fleet_machine_id = machine_id_fleet_config_filename(ctx);
+    if (fleet_machine_id == NULL) {
+        flb_plg_error(ctx->ins, "unable to allocate machine id file");
+        return NULL;
+    }
+
+    /* check if the file exists first, if it does not we generate a UUID */
+    if (flb_access(fleet_machine_id, F_OK) != 0) {
+
+        /* create new UUID for fleet */
+        buf = generate_uuid();
+        if( buf == NULL ) {
+            flb_plg_error(ctx->ins, "failed to create uuid for fleet machine id");
+            flb_sds_destroy(fleet_machine_id);
+            return NULL;
+        }
+        flb_plg_info(ctx->ins, "generated UUID for machine ID: %s", buf);
+
+        /* write uuid to file */
+        if (write_uuid_to_file(fleet_machine_id, buf ) != FLB_TRUE) {
+            flb_plg_error(ctx->ins, "failed to write fleet machine id file: %s", fleet_machine_id);
+            flb_free(buf);
+            flb_sds_destroy(fleet_machine_id);
+            return NULL;
+        }
+
+        flb_free(buf);
+        buf = NULL;
+
+        flb_plg_info(ctx->ins, "written machine ID to file: %s", fleet_machine_id);
+    }
+
+    /* now check file exists (it always should) and read from it */
+    if (flb_access(fleet_machine_id, F_OK) == 0) {
+        ret = flb_utils_read_file_wrapper(fleet_machine_id, &buf, &blen);
+        if (ret != 0) {
+            flb_plg_error(ctx->ins, "failed to read fleet machine id file: %s", fleet_machine_id);
+            flb_sds_destroy(fleet_machine_id);
+            return NULL;
+        }
+        flb_plg_info(ctx->ins, "read UUID (%s) from file: %s", buf, fleet_machine_id);
+    }
+    else { /* fall back to machine-id */
+        flb_plg_warn(ctx->ins, "unable to get uuid from file (%s) so falling back to machine id", fleet_machine_id);
+        ret = flb_utils_get_machine_id(&buf, &blen);
+    }
+
+    /* Clean up no longer required filename */
+    flb_sds_destroy(fleet_machine_id);
+#endif
 
     if (ret == -1) {
         flb_plg_error(ctx->ins, "could not obtain machine id");
@@ -520,13 +698,13 @@ static struct flb_config_map config_map[] = {
     },
 
     {
-     FLB_CONFIG_MAP_STR, "calyptia_host", "cloud-api.calyptia.com",
+     FLB_CONFIG_MAP_STR, "calyptia_host", DEFAULT_CALYPTIA_HOST,
      0, FLB_TRUE, offsetof(struct calyptia, cloud_host),
      ""
     },
 
     {
-     FLB_CONFIG_MAP_STR, "calyptia_port", "443",
+     FLB_CONFIG_MAP_STR, "calyptia_port", DEFAULT_CALYPTIA_PORT,
      0, FLB_TRUE, offsetof(struct calyptia, cloud_port),
      ""
     },
@@ -559,7 +737,7 @@ static struct flb_config_map config_map[] = {
      "Fleet id to be used when registering agent in a fleet"
     },
     {
-     FLB_CONFIG_MAP_STR, "fleet.config_dir", NULL,
+     FLB_CONFIG_MAP_STR, "fleet.config_dir", FLEET_DEFAULT_CONFIG_DIR,
      0, FLB_TRUE, offsetof(struct calyptia, fleet_config_dir),
      "Base path for the configuration directory."
     },

plugins/custom_calyptia/calyptia.h

@@ -20,6 +20,8 @@
 #ifndef FLB_CALYPTIA_H
 #define FLB_CALYPTIA_H
 
+#include <stdbool.h>
+
 struct calyptia {
     /* config map options */
     flb_sds_t api_key;
@@ -57,4 +59,10 @@ struct calyptia {
 };
 
 int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet);
+flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname);
+flb_sds_t get_machine_id(struct calyptia *ctx);
+
+/* These are unique to the agent rather than the fleet */
+#define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id")
+
 #endif /* FLB_CALYPTIA_H */