From 3e96ec59609a3a6013fa74c945a5775b1a1530e5 Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 5 Dec 2024 16:30:35 +0000 Subject: [PATCH] calyptia: generate machine id for fleet agents (#9691) * calyptia: generate machine id for fleet agents Signed-off-by: Patrick Stephens * calyptia: add missing include Signed-off-by: Patrick Stephens * calyptia: add missing include Signed-off-by: Patrick Stephens * tests: resolve conflicting definitions Signed-off-by: Patrick Stephens * calyptia: resolve missing bool type Signed-off-by: Patrick Stephens * calyptia: resolve missing header Signed-off-by: Patrick Stephens * calyptia: resolve duplicate symbols for tests Signed-off-by: Patrick Stephens * calyptia: resolve function pointer usage Signed-off-by: Patrick Stephens * calyptia: update codeowners Signed-off-by: Patrick Stephens * custom_calyptia: tests: Add missing declarations To link those missing declaration functions, we need to define for stub in the custom_calyptia testing files. Signed-off-by: Hiroshi Hatake * custom_calyptia: Remove duplicated function declarations Signed-off-by: Hiroshi Hatake * tests: add calyptia machine id generation tests Signed-off-by: Patrick Stephens * tests: add missing header Signed-off-by: Patrick Stephens * tests: fix typos Signed-off-by: Patrick Stephens * tests: fix typos Signed-off-by: Patrick Stephens * tests: fix memory issues Signed-off-by: Patrick Stephens * tests: fix location Signed-off-by: Patrick Stephens * tests: fix destroy usage Signed-off-by: Patrick Stephens * tests: fix destroy usage Signed-off-by: Patrick Stephens --------- Signed-off-by: Patrick Stephens Signed-off-by: Hiroshi Hatake Co-authored-by: Hiroshi Hatake --- CODEOWNERS | 8 +- Vagrantfile | 2 +- include/CMakeLists.txt | 6 + .../fluent-bit/calyptia/calyptia_constants.h | 68 ++++++ include/fluent-bit/flb_utils.h | 1 + lib/cprofiles/include/cprofiles/cprof_info.h | 5 +- plugins/custom_calyptia/calyptia.c | 202 ++++++++++++++++-- plugins/custom_calyptia/calyptia.h | 8 + plugins/in_calyptia_fleet/in_calyptia_fleet.c | 95 ++------ plugins/out_calyptia/calyptia.c | 36 ++-- plugins/out_calyptia/calyptia.h | 26 +-- src/flb_utils.c | 52 +++++ tests/runtime/custom_calyptia_input_test.c | 114 +++++++++- tests/runtime/custom_calyptia_test.c | 6 + 14 files changed, 489 insertions(+), 140 deletions(-) create mode 100644 include/fluent-bit/calyptia/calyptia_constants.h diff --git a/CODEOWNERS b/CODEOWNERS index 2b5b404a281..d52de3f298b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -91,6 +91,8 @@ # Devcontainer /.devcontainer @patrick-stephens @niedbalski @edsiper -# Calytia Fleet -/plugins/custom_calyptia/ @pwhelan -/plugins/in_calyptia_fleet/ @pwhelan +# Calyptia Fleet +/include/fluent-bit/calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/custom_calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/custom_calyptia/ @pwhelan @patrick-stephens @niedbalski +/plugins/out_calyptia/ @pwhelan @patrick-stephens @niedbalski diff --git a/Vagrantfile b/Vagrantfile index 7f74824cb34..b18ea9795ee 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,7 +28,7 @@ Vagrant.configure("2") do |config| # Main build apt-get install --yes build-essential cmake dh-make git make openssl pkg-config tar # Dependencies - apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5 + apt-get install --yes libssl3 libssl-dev libsasl2-dev pkg-config libsystemd-dev zlib1g-dev libpq-dev postgresql-server-dev-all flex bison libyaml-dev libpq5 libbpf-dev # Debug apt-get install --yes gdb valgrind diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 38f1b0dccd5..30ec68744a8 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -10,6 +10,12 @@ install(FILES ${headers} COMPONENT headers PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) +file(GLOB headers "fluent-bit/calyptia/*.h") +install(FILES ${headers} + DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit + COMPONENT headers + PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) + file(GLOB headers "fluent-bit/config_format/*.h") install(FILES ${headers} DESTINATION ${FLB_INSTALL_INCLUDEDIR}/fluent-bit/config_format/ diff --git a/include/fluent-bit/calyptia/calyptia_constants.h b/include/fluent-bit/calyptia/calyptia_constants.h new file mode 100644 index 00000000000..f2271b87a67 --- /dev/null +++ b/include/fluent-bit/calyptia/calyptia_constants.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FLB_CALYPTIA_CONSTANTS_H +#define FLB_CALYPTIA_CONSTANTS_H + +/* End point */ +#define DEFAULT_CALYPTIA_HOST "cloud-api.calyptia.com" +#define DEFAULT_CALYPTIA_PORT "443" + +/* HTTP action types */ +#define CALYPTIA_ACTION_REGISTER 0 +#define CALYPTIA_ACTION_PATCH 1 +#define CALYPTIA_ACTION_METRICS 2 +#define CALYPTIA_ACTION_TRACE 3 + +/* Endpoints */ +#define CALYPTIA_ENDPOINT_CREATE "/v1/agents" +#define CALYPTIA_ENDPOINT_PATCH "/v1/agents/%s" +#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics" +#define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" + +#define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini" +#define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files" + +/* Storage */ +#define CALYPTIA_SESSION_FILE "session.CALYPTIA" + +/* Headers */ +#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" +#define CALYPTIA_HEADERS_AGENT_TOKEN "X-Agent-Token" +#define CALYPTIA_HEADERS_CTYPE "Content-Type" +#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" +#define CALYPTIA_HEADERS_CTYPE_MSGPACK "application/x-msgpack" + +#ifndef FLB_SYSTEM_WINDOWS +#define FLEET_DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet" +#else +#define FLEET_DEFAULT_CONFIG_DIR NULL +#endif + +#ifndef PATH_SEPARATOR +#ifndef FLB_SYSTEM_WINDOWS +#define PATH_SEPARATOR "/" +#else +#define PATH_SEPARATOR "\\" +#endif +#endif /* PATH_SEPARATOR */ + +#define CALYPTIA_MAX_DIR_SIZE 4096 + +#endif /* FLB_CALYPTIA_CONSTANTS_H */ diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index b1f06592278..a0719c53e50 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -74,5 +74,6 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size); void flb_utils_set_plugin_string_property(const char *name, flb_sds_t *field_storage, flb_sds_t new_value); +int flb_utils_mkdir(const char *dir, int perms); #endif diff --git a/lib/cprofiles/include/cprofiles/cprof_info.h b/lib/cprofiles/include/cprofiles/cprof_info.h index d0e26fcfbfe..10f06c905cd 100644 --- a/lib/cprofiles/include/cprofiles/cprof_info.h +++ b/lib/cprofiles/include/cprofiles/cprof_info.h @@ -20,12 +20,9 @@ #ifndef CPROF_INFO_H #define CPROF_INFO_H -#define CPROF_SOURCE_DIR "/Users/leonardo/Work/Calyptia/fluent-bit" +#define CPROF_SOURCE_DIR "/src/fluent-bit" /* General flags set by /CMakeLists.txt */ -#ifndef CPROF_HAVE_SANITIZE_ADDRESS -#define CPROF_HAVE_SANITIZE_ADDRESS -#endif #ifndef CPROF_HAVE_TIMESPEC_GET #define CPROF_HAVE_TIMESPEC_GET #endif diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index bfbb42f4767..dba68a25f10 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -27,11 +27,25 @@ #include #include #include - +#include +#include +#include +#include #include +#include + #include "calyptia.h" +#define UUID_BUFFER_SIZE 38 /* Maximum length of UUID string + null terminator */ + +/* Function wrappers to enable mocking for unit test filesystem access */ +int (*flb_access)(const char *pathname, int mode) = access; +int (*flb_open)(const char *pathname, int flags, ...) = open; +ssize_t (*flb_write)(int fd, const void *buf, size_t count) = write; +int (*flb_close)(int fd) = close; +int (*flb_utils_read_file_wrapper)(char *path, char **out_buf, size_t *out_size) = flb_utils_read_file; + /* * Check if the key belongs to a sensitive data field, if so report it. We never * share any sensitive data. @@ -217,16 +231,13 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance * flb_input_set_property(fleet, "api_key", ctx->api_key); flb_input_set_property(fleet, "host", ctx->cloud_host); flb_input_set_property(fleet, "port", ctx->cloud_port); + flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); /* Set TLS properties */ flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off"); flb_input_set_property(fleet, "tls.verify", ctx->cloud_tls_verify == 1 ? "on" : "off"); /* Optional configurations */ - if (ctx->fleet_config_dir) { - flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); - } - if (ctx->fleet_max_http_buffer_size) { flb_input_set_property(fleet, "max_http_buffer_size", ctx->fleet_max_http_buffer_size); } @@ -376,15 +387,182 @@ static flb_sds_t sha256_to_hex(unsigned char *sha256) return hex; } -static flb_sds_t get_machine_id(struct calyptia *ctx) +static flb_sds_t generate_base_agent_directory(struct calyptia *ctx, flb_sds_t *fleet_dir) { - int ret; - char *buf; - size_t blen; + flb_sds_t ret = NULL; + + if (ctx == NULL || fleet_dir == NULL) { + return NULL; + } + + if (*fleet_dir == NULL) { + *fleet_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); + if (*fleet_dir == NULL) { + return NULL; + } + } + + ret = flb_sds_printf(fleet_dir, "%s", ctx->fleet_config_dir); + if (ret == NULL) { + flb_sds_destroy(*fleet_dir); + return NULL; + } + + return ret; +} + +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname) +{ + flb_sds_t cfgname = NULL; + flb_sds_t ret; + + if (ctx == NULL || fname == NULL) { + return NULL; + } + + if (generate_base_agent_directory(ctx, &cfgname) == NULL) { + return NULL; + } + + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + if (ret == NULL) { + flb_sds_destroy(cfgname); + return NULL; + } + + return cfgname; +} + +static char* generate_uuid() { + char* uuid = flb_malloc(UUID_BUFFER_SIZE); + if (!uuid) { + flb_errno(); + return NULL; + } + + /* create new UUID for fleet */ + if (flb_utils_uuid_v4_gen(uuid) != 0 || strlen(uuid) == 0) { + flb_free(uuid); + return NULL; + } + return uuid; +} + +static int write_uuid_to_file(flb_sds_t fleet_machine_id, char* uuid) { + int fd; + size_t uuid_len; + + if (fleet_machine_id == NULL || uuid == NULL) { + return FLB_FALSE; + } + + /* write uuid to file */ + fd = flb_open(fleet_machine_id, O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) { + return FLB_FALSE; + } + + uuid_len = strlen(uuid); + + if (flb_write(fd, uuid, uuid_len) != uuid_len) { + flb_close(fd); + return FLB_FALSE; + } + + flb_close(fd); + return FLB_TRUE; +} + +static int create_agent_directory(struct calyptia *ctx) +{ + if( ctx == NULL ) { + return -1; + } + + /* If it exists just return */ + if (access(ctx->fleet_config_dir, F_OK) == 0) { + return 0; + } + + /* Create the directory if it does not exist */ + if (flb_utils_mkdir(ctx->fleet_config_dir, 0700) != 0) { + flb_plg_error(ctx->ins, "failed to create directory: %s", ctx->fleet_config_dir); + return -1; + } + + return 0; +} + +flb_sds_t get_machine_id(struct calyptia *ctx) +{ + int ret = -1; + char *buf = NULL; + size_t blen = 0; unsigned char sha256_buf[64] = {0}; +#if defined(FLB_SYSTEM_WINDOWS) /* retrieve raw machine id */ ret = flb_utils_get_machine_id(&buf, &blen); +#else + /* /etc/machine-id is not guaranteed to be unique so we generate one */ + flb_sds_t fleet_machine_id = NULL; + + /** ensure we have the directory created */ + if (create_agent_directory(ctx) != 0) { + return NULL; + } + + /** now get the agent filename */ + fleet_machine_id = machine_id_fleet_config_filename(ctx); + if (fleet_machine_id == NULL) { + flb_plg_error(ctx->ins, "unable to allocate machine id file"); + return NULL; + } + + /* check if the file exists first, if it does not we generate a UUID */ + if (flb_access(fleet_machine_id, F_OK) != 0) { + + /* create new UUID for fleet */ + buf = generate_uuid(); + if( buf == NULL ) { + flb_plg_error(ctx->ins, "failed to create uuid for fleet machine id"); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + flb_plg_info(ctx->ins, "generated UUID for machine ID: %s", buf); + + /* write uuid to file */ + if (write_uuid_to_file(fleet_machine_id, buf ) != FLB_TRUE) { + flb_plg_error(ctx->ins, "failed to write fleet machine id file: %s", fleet_machine_id); + flb_free(buf); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + + flb_free(buf); + buf = NULL; + + flb_plg_info(ctx->ins, "written machine ID to file: %s", fleet_machine_id); + } + + /* now check file exists (it always should) and read from it */ + if (flb_access(fleet_machine_id, F_OK) == 0) { + ret = flb_utils_read_file_wrapper(fleet_machine_id, &buf, &blen); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to read fleet machine id file: %s", fleet_machine_id); + flb_sds_destroy(fleet_machine_id); + return NULL; + } + flb_plg_info(ctx->ins, "read UUID (%s) from file: %s", buf, fleet_machine_id); + } + else { /* fall back to machine-id */ + flb_plg_warn(ctx->ins, "unable to get uuid from file (%s) so falling back to machine id", fleet_machine_id); + ret = flb_utils_get_machine_id(&buf, &blen); + } + + /* Clean up no longer required filename */ + flb_sds_destroy(fleet_machine_id); +#endif if (ret == -1) { flb_plg_error(ctx->ins, "could not obtain machine id"); @@ -520,13 +698,13 @@ static struct flb_config_map config_map[] = { }, { - FLB_CONFIG_MAP_STR, "calyptia_host", "cloud-api.calyptia.com", + FLB_CONFIG_MAP_STR, "calyptia_host", DEFAULT_CALYPTIA_HOST, 0, FLB_TRUE, offsetof(struct calyptia, cloud_host), "" }, { - FLB_CONFIG_MAP_STR, "calyptia_port", "443", + FLB_CONFIG_MAP_STR, "calyptia_port", DEFAULT_CALYPTIA_PORT, 0, FLB_TRUE, offsetof(struct calyptia, cloud_port), "" }, @@ -559,7 +737,7 @@ static struct flb_config_map config_map[] = { "Fleet id to be used when registering agent in a fleet" }, { - FLB_CONFIG_MAP_STR, "fleet.config_dir", NULL, + FLB_CONFIG_MAP_STR, "fleet.config_dir", FLEET_DEFAULT_CONFIG_DIR, 0, FLB_TRUE, offsetof(struct calyptia, fleet_config_dir), "Base path for the configuration directory." }, diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index b4313f51182..5da14d80ace 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,6 +20,8 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H +#include + struct calyptia { /* config map options */ flb_sds_t api_key; @@ -57,4 +59,10 @@ struct calyptia { }; int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); +flb_sds_t get_machine_id(struct calyptia *ctx); + +/* These are unique to the agent rather than the fleet */ +#define machine_id_fleet_config_filename(a) agent_config_filename((a), "machine-id") + #endif /* FLB_CALYPTIA_H */ diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.c b/plugins/in_calyptia_fleet/in_calyptia_fleet.c index 28e3fc8719c..5a423856c32 100644 --- a/plugins/in_calyptia_fleet/in_calyptia_fleet.c +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.c @@ -38,6 +38,9 @@ #include #include #include +#include + +#include /* Glob support */ #ifndef _MSC_VER @@ -50,26 +53,15 @@ #define PATH_MAX MAX_PATH #endif -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" +#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" +#define CALYPTIA_HEADERS_CTYPE "Content-Type" +#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" #define DEFAULT_INTERVAL_SEC "15" #define DEFAULT_INTERVAL_NSEC "0" #define DEFAULT_MAX_HTTP_BUFFER_SIZE "10485760" -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -#ifndef _WIN32 -#define PATH_SEPARATOR "/" -#define DEFAULT_CONFIG_DIR "/tmp/calyptia-fleet" -#else -#define DEFAULT_CONFIG_DIR NULL -#define PATH_SEPARATOR "\\" -#endif - struct flb_in_calyptia_fleet_config { /* Time interval check */ int interval_sec; @@ -920,7 +912,7 @@ static struct flb_http_client *fleet_http_do(struct flb_in_calyptia_fleet_config flb_http_buffer_size(client, ctx->max_http_buffer_size); flb_http_add_header(client, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, + CALYPTIA_HEADERS_PROJECT, sizeof(CALYPTIA_HEADERS_PROJECT) - 1, ctx->api_key, flb_sds_len(ctx->api_key)); ret = flb_http_do(client, &b_sent); @@ -1087,56 +1079,6 @@ static int get_calyptia_file(struct flb_in_calyptia_fleet_config *ctx, return ret; } -#ifdef FLB_SYSTEM_WINDOWS -#define _mkdir(a, b) mkdir(a) -#else -#define _mkdir(a, b) mkdir(a, b) -#endif - -/* recursively create directories, based on: - * https://stackoverflow.com/a/2336245 - * who found it at: - * http://nion.modprobe.de/blog/archives/357-Recursive-directory-creation.html - */ -static int __mkdir(const char *dir, int perms) { - char tmp[255]; - char *ptr = NULL; - size_t len; - int ret; - - ret = snprintf(tmp, sizeof(tmp),"%s",dir); - if (ret > sizeof(tmp)) { - flb_error("directory too long for __mkdir: %s", dir); - return -1; - } - - len = strlen(tmp); - - if (tmp[len - 1] == PATH_SEPARATOR[0]) { - tmp[len - 1] = 0; - } - -#ifndef FLB_SYSTEM_WINDOWS - for (ptr = tmp + 1; *ptr; ptr++) { -#else - for (ptr = tmp + 3; *ptr; ptr++) { -#endif - - if (*ptr == PATH_SEPARATOR[0]) { - *ptr = 0; - if (access(tmp, F_OK) != 0) { - ret = _mkdir(tmp, perms); - if (ret != 0) { - return ret; - } - } - *ptr = PATH_SEPARATOR[0]; - } - } - - return _mkdir(tmp, perms); -} - #ifndef _WIN32 static struct cfl_array *read_glob(const char *path) { @@ -1839,7 +1781,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } - flb_sds_printf(&ctx->fleet_url, "/v1/fleets/%s/config?format=ini", ctx->fleet_id); + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); } if (ctx->fleet_files_url == NULL) { @@ -1849,7 +1791,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } - flb_sds_printf(&ctx->fleet_files_url, "/v1/fleets/%s/files", ctx->fleet_id); + flb_sds_printf(&ctx->fleet_files_url, CALYPTIA_ENDPOINT_FLEET_FILES, ctx->fleet_id); } create_fleet_header(ctx); @@ -1920,7 +1862,7 @@ static int create_fleet_directory(struct flb_in_calyptia_fleet_config *ctx) flb_sds_t myfleetdir = NULL; if (access(ctx->config_dir, F_OK) != 0) { - if (__mkdir(ctx->config_dir, 0700) != 0) { + if (flb_utils_mkdir(ctx->config_dir, 0700) != 0) { return -1; } } @@ -1931,7 +1873,7 @@ static int create_fleet_directory(struct flb_in_calyptia_fleet_config *ctx) } if (access(myfleetdir, F_OK) != 0) { - if (__mkdir(myfleetdir, 0700) !=0) { + if (flb_utils_mkdir(myfleetdir, 0700) !=0) { return -1; } } @@ -1970,18 +1912,19 @@ static flb_sds_t fleet_gendir(struct flb_in_calyptia_fleet_config *ctx, time_t t static int fleet_mkdir(struct flb_in_calyptia_fleet_config *ctx, time_t timestamp) { + int ret = -1; flb_sds_t fleetcurdir; fleetcurdir = fleet_gendir(ctx, timestamp); - if (fleetcurdir == NULL) { - return -1; + if (fleetcurdir != NULL) { + if (flb_utils_mkdir(fleetcurdir, 0700) == 0) { + ret = 0; + } + flb_sds_destroy(fleetcurdir); } - __mkdir(fleetcurdir, 0700); - flb_sds_destroy(fleetcurdir); - - return 0; + return ret; } static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx) @@ -2378,7 +2321,7 @@ static struct flb_config_map config_map[] = { "Calyptia Cloud API Key." }, { - FLB_CONFIG_MAP_STR, "config_dir", DEFAULT_CONFIG_DIR, + FLB_CONFIG_MAP_STR, "config_dir", FLEET_DEFAULT_CONFIG_DIR, 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, config_dir), "Base path for the configuration directory." }, diff --git a/plugins/out_calyptia/calyptia.c b/plugins/out_calyptia/calyptia.c index c16fd31d6ee..5f145075c1e 100644 --- a/plugins/out_calyptia/calyptia.c +++ b/plugins/out_calyptia/calyptia.c @@ -341,43 +341,43 @@ static int calyptia_http_do(struct flb_calyptia *ctx, struct flb_http_client *c, return FLB_ERROR; } flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_PROJECT, sizeof(CALYPTIA_H_PROJECT) - 1, + CALYPTIA_HEADERS_PROJECT, sizeof(CALYPTIA_HEADERS_PROJECT) - 1, ctx->api_key, flb_sds_len(ctx->api_key)); } else if (type == CALYPTIA_ACTION_PATCH) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } else if (type == CALYPTIA_ACTION_METRICS) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_MSGPACK, - sizeof(CALYPTIA_H_CTYPE_MSGPACK) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_MSGPACK, + sizeof(CALYPTIA_HEADERS_CTYPE_MSGPACK) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } #ifdef FLB_HAVE_CHUNK_TRACE else if (type == CALYPTIA_ACTION_TRACE) { flb_http_add_header(c, - CALYPTIA_H_CTYPE, sizeof(CALYPTIA_H_CTYPE) - 1, - CALYPTIA_H_CTYPE_JSON, sizeof(CALYPTIA_H_CTYPE_JSON) - 1); + CALYPTIA_HEADERS_CTYPE, sizeof(CALYPTIA_HEADERS_CTYPE) - 1, + CALYPTIA_HEADERS_CTYPE_JSON, sizeof(CALYPTIA_HEADERS_CTYPE_JSON) - 1); flb_http_add_header(c, - CALYPTIA_H_AGENT_TOKEN, - sizeof(CALYPTIA_H_AGENT_TOKEN) - 1, + CALYPTIA_HEADERS_AGENT_TOKEN, + sizeof(CALYPTIA_HEADERS_AGENT_TOKEN) - 1, ctx->agent_token, flb_sds_len(ctx->agent_token)); } #endif @@ -1059,13 +1059,13 @@ static void cb_calyptia_flush(struct flb_event_chunk *event_chunk, /* Configuration properties map */ static struct flb_config_map config_map[] = { { - FLB_CONFIG_MAP_STR, "cloud_host", CALYPTIA_HOST, + FLB_CONFIG_MAP_STR, "cloud_host", DEFAULT_CALYPTIA_HOST, 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_host), "", }, { - FLB_CONFIG_MAP_INT, "cloud_port", CALYPTIA_PORT, + FLB_CONFIG_MAP_INT, "cloud_port", DEFAULT_CALYPTIA_PORT, 0, FLB_TRUE, offsetof(struct flb_calyptia, cloud_port), "", }, diff --git a/plugins/out_calyptia/calyptia.h b/plugins/out_calyptia/calyptia.h index 0532e4a2b01..4da215d2206 100644 --- a/plugins/out_calyptia/calyptia.h +++ b/plugins/out_calyptia/calyptia.h @@ -25,31 +25,7 @@ #include #include -/* End point */ -#define CALYPTIA_HOST "cloud-api.calyptia.com" -#define CALYPTIA_PORT "443" - -/* HTTP action types */ -#define CALYPTIA_ACTION_REGISTER 0 -#define CALYPTIA_ACTION_PATCH 1 -#define CALYPTIA_ACTION_METRICS 2 -#define CALYPTIA_ACTION_TRACE 3 - -/* Endpoints */ -#define CALYPTIA_ENDPOINT_CREATE "/v1/agents" -#define CALYPTIA_ENDPOINT_PATCH "/v1/agents/%s" -#define CALYPTIA_ENDPOINT_METRICS "/v1/agents/%s/metrics" -#define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" - -/* Storage */ -#define CALYPTIA_SESSION_FILE "session.CALYPTIA" - -/* Headers */ -#define CALYPTIA_H_PROJECT "X-Project-Token" -#define CALYPTIA_H_AGENT_TOKEN "X-Agent-Token" -#define CALYPTIA_H_CTYPE "Content-Type" -#define CALYPTIA_H_CTYPE_JSON "application/json" -#define CALYPTIA_H_CTYPE_MSGPACK "application/x-msgpack" +#include struct flb_calyptia { /* config map */ diff --git a/src/flb_utils.c b/src/flb_utils.c index 211632260e3..3aa4412086b 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -37,6 +37,8 @@ #include #include +#include + #ifdef FLB_HAVE_AWS_ERROR_REPORTER #include @@ -1636,3 +1638,53 @@ void flb_utils_set_plugin_string_property(const char *name, *field_storage = new_value; } + +#ifdef FLB_SYSTEM_WINDOWS +#define _mkdir(a, b) mkdir(a) +#else +#define _mkdir(a, b) mkdir(a, b) +#endif + +/* recursively create directories, based on: + * https://stackoverflow.com/a/2336245 + * who found it at: + * http://nion.modprobe.de/blog/archives/357-Recursive-directory-creation.html + */ +int flb_utils_mkdir(const char *dir, int perms) { + char tmp[CALYPTIA_MAX_DIR_SIZE]; + char *ptr = NULL; + size_t len; + int ret; + + ret = snprintf(tmp, sizeof(tmp),"%s",dir); + if (ret < 0 || ret >= sizeof(tmp)) { + flb_error("directory too long for flb_utils_mkdir: %s", dir); + return -1; + } + + len = strlen(tmp); + /* len == ret but verifying index is valid */ + if ( len > 0 && tmp[len - 1] == PATH_SEPARATOR[0]) { + tmp[len - 1] = 0; + } + +#ifndef FLB_SYSTEM_WINDOWS + for (ptr = tmp + 1; *ptr; ptr++) { +#else + for (ptr = tmp + 3; *ptr; ptr++) { +#endif + + if (*ptr == PATH_SEPARATOR[0]) { + *ptr = 0; + if (access(tmp, F_OK) != 0) { + ret = _mkdir(tmp, perms); + if (ret != 0) { + return ret; + } + } + *ptr = PATH_SEPARATOR[0]; + } + } + + return _mkdir(tmp, perms); +} diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index ffc774c83d9..5cf023afa76 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -1,13 +1,26 @@ #include #include #include -#include #include #include #include +#include +#include +#include +#include #include "flb_tests_runtime.h" #include "../../plugins/custom_calyptia/calyptia.h" +const char *flb_input_get_property(const char *key, + struct flb_input_instance *ins); +struct flb_input_instance *flb_input_new(struct flb_config *config, + const char *input, void *data, + int public_only); +void flb_input_instance_destroy(struct flb_input_instance *ins); + +flb_sds_t agent_config_filename(struct calyptia *ctx, char *fname); +flb_sds_t get_machine_id(struct calyptia *ctx); + /* Test context structure */ struct test_context { struct calyptia *ctx; @@ -165,8 +178,107 @@ void test_set_fleet_input_properties() cleanup_test_context(t_ctx); } +static struct test_context * update_config_dir(struct test_context * t_ctx, const char* new_config_dir) { + if (!t_ctx || !new_config_dir) { + return NULL; + } + + if (t_ctx->ctx) { + if (t_ctx->ctx->fleet_config_dir) { + flb_free(t_ctx->ctx->fleet_config_dir); + } + t_ctx->ctx->fleet_config_dir = flb_strdup(new_config_dir); + return t_ctx; + } + + cleanup_test_context(t_ctx); + return NULL; +} + +static void test_calyptia_machine_id_generation() { + struct test_context *t_ctx = init_test_context(); + TEST_CHECK(t_ctx != NULL); + + /* Set config directory to default */ + t_ctx = update_config_dir(t_ctx, FLEET_DEFAULT_CONFIG_DIR); + TEST_CHECK(t_ctx != NULL); + + /* Test setting properties */ + int ret = set_fleet_input_properties(t_ctx->ctx, t_ctx->fleet); + TEST_CHECK(ret == 0); + + /* Verify properties were set correctly */ + const char *value; + + /* Check config_dir */ + value = flb_input_get_property("config_dir", t_ctx->fleet); + TEST_CHECK(value != NULL); + TEST_MSG("config_dir expected=%s got=%s", FLEET_DEFAULT_CONFIG_DIR, value); + TEST_CHECK(value && strcmp(value, FLEET_DEFAULT_CONFIG_DIR) == 0); + + /** + * Initial generation should create a new UUID + * Subsequent generation should reuse the previous one + * Repeat with custom directory to confirm that works too + */ + char expectedValue[CALYPTIA_MAX_DIR_SIZE]; + ret = sprintf(expectedValue, "%s/machine-id.conf", FLEET_DEFAULT_CONFIG_DIR); + TEST_CHECK(ret > 0); + + flb_sds_t filename = machine_id_fleet_config_filename(t_ctx->ctx); + TEST_CHECK(filename != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, filename); + TEST_CHECK(filename && strcmp(filename, expectedValue) == 0); + flb_sds_destroy(filename); + + /* generate a new machine ID and verify it is not null then store for later use */ + flb_sds_t machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(machine_id != NULL); + + /* repeat to confirm existing UUID is maintained */ + flb_sds_t new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(value && strcmp(new_machine_id, machine_id) == 0); + flb_sds_destroy(new_machine_id); + + /* repeat with new config directory */ + t_ctx = update_config_dir(t_ctx, "/tmp/config/fleet"); + TEST_CHECK(t_ctx != NULL); + + /* check we use the new directory for the filename */ + ret = sprintf(expectedValue, "/tmp/config/fleet/machine-id.conf"); + TEST_CHECK(ret > 0); + + filename = machine_id_fleet_config_filename(t_ctx->ctx); + TEST_CHECK(filename != NULL); + TEST_MSG("machine_id filename expected=%s got=%s", expectedValue, filename); + TEST_CHECK(filename && strcmp(filename, expectedValue) == 0); + flb_sds_destroy(filename); + + /* check we generate a new value */ + new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id did not change, expected!=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) != 0); + + flb_sds_destroy(machine_id); + machine_id = new_machine_id; + + /* repeat to confirm existing UUID is maintained */ + new_machine_id = get_machine_id(t_ctx->ctx); + TEST_CHECK(new_machine_id != NULL); + TEST_MSG("machine_id changed, expected=%s got=%s", machine_id, new_machine_id); + TEST_CHECK(new_machine_id && strcmp(new_machine_id, machine_id) == 0); + + flb_sds_destroy(new_machine_id); + flb_sds_destroy(machine_id); + cleanup_test_context(t_ctx); +} + /* Define test list */ TEST_LIST = { {"set_fleet_input_properties", test_set_fleet_input_properties}, + {"machine_id_generation", test_calyptia_machine_id_generation}, {NULL, NULL} }; \ No newline at end of file diff --git a/tests/runtime/custom_calyptia_test.c b/tests/runtime/custom_calyptia_test.c index 7a13fa48dd1..00d65fd3a36 100644 --- a/tests/runtime/custom_calyptia_test.c +++ b/tests/runtime/custom_calyptia_test.c @@ -5,6 +5,12 @@ #include #include "flb_tests_runtime.h" +const char *flb_input_get_property(const char *key, + struct flb_input_instance *ins); +struct flb_input_instance *flb_input_new(struct flb_config *config, + const char *input, void *data, + int public_only); + flb_sds_t custom_calyptia_pipeline_config_get(struct flb_config *ctx); void flb_custom_calyptia_pipeline_config_get_test()