From 9a263e839f686950121e51a50c240045ee8eab69 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 11 Jun 2023 12:03:08 +0900 Subject: [PATCH 01/11] tests: internal: log_event_encoder: add test code Signed-off-by: Takahiro Yamashita --- tests/internal/CMakeLists.txt | 1 + tests/internal/log_event_encoder.c | 663 +++++++++++++++++++++++++++++ 2 files changed, 664 insertions(+) create mode 100644 tests/internal/log_event_encoder.c diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index b92b999cec3..5ddd23afe43 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -38,6 +38,7 @@ set(UNIT_TESTS_FILES parser_logfmt.c env.c log.c + log_event_encoder.c processor.c ) diff --git a/tests/internal/log_event_encoder.c b/tests/internal/log_event_encoder.c new file mode 100644 index 00000000000..564947f9a74 --- /dev/null +++ b/tests/internal/log_event_encoder.c @@ -0,0 +1,663 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2023 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "flb_tests_internal.h" + +static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) +{ + int ret = -1; + + if (str == NULL) { + flb_error("str is NULL"); + return -1; + } + + switch (obj.type) { + case MSGPACK_OBJECT_STR: + if (obj.via.str.size != str_len) { + return -1; + } + ret = strncmp(str, obj.via.str.ptr, str_len); + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + { + unsigned long val = strtoul(str, NULL, 10); + if (val == (unsigned long)obj.via.u64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + { + long long val = strtoll(str, NULL, 10); + if (val == (unsigned long)obj.via.i64) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + { + double val = strtod(str, NULL); + if (fabs(val - obj.via.f64) < DBL_EPSILON) { + ret = 0; + } + } + break; + case MSGPACK_OBJECT_BOOLEAN: + if (obj.via.boolean) { + if (str_len != 4 /*true*/) { + return -1; + } + ret = strncasecmp(str, "true", 4); + } + else { + if (str_len != 5 /*false*/) { + return -1; + } + ret = strncasecmp(str, "false", 5); + } + break; + default: + flb_error("not supported"); + } + + return ret; +} + +struct str_list { + size_t size; + char **lists; +}; + +static int compare_msgpack_map(msgpack_object *map, struct str_list *l) +{ + int map_size; + int i_map; + int i_list; + int num = 0; + + if (!TEST_CHECK(map->type == MSGPACK_OBJECT_MAP)) { + TEST_MSG("type is not map. type = %d", map->type); + return -1; + } + + map_size = map->via.map.size; + for (i_map=0; i_mapvia.map.ptr[i_map].key.type == MSGPACK_OBJECT_STR)) { + TEST_MSG("key is not string. type =%d", map->via.map.ptr[i_map].key.type); + continue; + } + for (i_list=0; i_list< l->size/2; i_list++) { + if (msgpack_strncmp(l->lists[i_list*2], strlen(l->lists[i_list*2]), + map->via.map.ptr[i_map].key) == 0 && + msgpack_strncmp(l->lists[i_list*2+1], strlen(l->lists[i_list*2+1]), + map->via.map.ptr[i_map].val) == 0) { + num++; + } + } + } + if (!TEST_CHECK(num == l->size/2)) { + msgpack_object_print(stdout, *map); + putchar('\n'); + TEST_MSG("compare failed. matched_num=%d expect=%lu", num, l->size/2); + return -1; + } + + return 0; +} + +static int compare_msgpack_format_fluentbit_v2(void *msgpack_data, size_t msgpack_size, + struct str_list *metadata, + struct str_list *body) +{ + msgpack_unpacked result; + msgpack_object obj; + msgpack_object root; + size_t off = 0; + int ret; + + if (!TEST_CHECK(msgpack_data != NULL)) { + TEST_MSG("msgpack_data is NULL"); + return -1; + } + else if (!TEST_CHECK(msgpack_size > 0)) { + TEST_MSG("msgpack_size is 0"); + return -1; + } + + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, msgpack_data, msgpack_size, &off) == MSGPACK_UNPACK_SUCCESS) { + root = result.data; + /* + msgpack_object_print(stdout, obj); + */ + + /* format v2: [[timestamp, {metadata}], {record}]*/ + + if (!TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY)) { + TEST_MSG("type is not array. type = %d", root.type); + msgpack_unpacked_destroy(&result); + return -1; + } + if (!TEST_CHECK(root.via.array.size == 2)) { + TEST_MSG("array size error. size = %d", root.via.array.size); + msgpack_unpacked_destroy(&result); + return -1; + } + + obj = root.via.array.ptr[0]; /* [timestamp, {metadata}] */ + if (!TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY)) { + TEST_MSG("type is not array. type = %d", root.type); + msgpack_unpacked_destroy(&result); + return -1; + } + if (!TEST_CHECK(root.via.array.size == 2)) { + TEST_MSG("array size error. size = %d", root.via.array.size); + msgpack_unpacked_destroy(&result); + return -1; + } + + obj = root.via.array.ptr[0].via.array.ptr[0]; /* timestamp */ + if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_EXT || + obj.type == MSGPACK_OBJECT_POSITIVE_INTEGER)) { + TEST_MSG("timestamp format error. type = %d", obj.type); + msgpack_unpacked_destroy(&result); + return -1; + } + obj = root.via.array.ptr[0].via.array.ptr[1]; /* metadata */ + if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_MAP)) { + TEST_MSG("type is not map. type = %d", obj.type); + msgpack_unpacked_destroy(&result); + return -1; + } + if (metadata != NULL) { + ret = compare_msgpack_map(&obj, metadata); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare_msgpack_body failed"); + msgpack_unpacked_destroy(&result); + return -1; + } + } + else if (!TEST_CHECK(obj.via.map.size == 0)) { + TEST_MSG("map size error. size = %d", root.via.map.size); + msgpack_unpacked_destroy(&result); + return -1; + } + + obj = root.via.array.ptr[1]; /* {record} */ + ret = compare_msgpack_map(&obj, body); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare_msgpack_body failed"); + msgpack_unpacked_destroy(&result); + return -1; + } + + } + msgpack_unpacked_destroy(&result); + + return 0; +} + +static int compare_msgpack_format_fluentbit_v1(void *msgpack_data, size_t msgpack_size, + struct str_list *body) +{ + msgpack_unpacked result; + msgpack_object obj; + msgpack_object root; + size_t off = 0; + int ret; + + if (!TEST_CHECK(msgpack_data != NULL)) { + TEST_MSG("msgpack_data is NULL"); + return -1; + } + else if (!TEST_CHECK(msgpack_size > 0)) { + TEST_MSG("msgpack_size is 0"); + return -1; + } + + msgpack_unpacked_init(&result); + while (msgpack_unpack_next(&result, msgpack_data, msgpack_size, &off) == MSGPACK_UNPACK_SUCCESS) { + root = result.data; + /* + msgpack_object_print(stdout, obj); + */ + + /* format v1: [timestamp, {record}]*/ + + if (!TEST_CHECK(root.type == MSGPACK_OBJECT_ARRAY)) { + TEST_MSG("type is not array. type = %d", root.type); + msgpack_unpacked_destroy(&result); + return -1; + } + if (!TEST_CHECK(root.via.array.size == 2)) { + TEST_MSG("array size error. size = %d", root.via.array.size); + msgpack_unpacked_destroy(&result); + return -1; + } + + obj = root.via.array.ptr[0]; /* timestamp */ + if (!TEST_CHECK(obj.type == MSGPACK_OBJECT_EXT || + obj.type == MSGPACK_OBJECT_POSITIVE_INTEGER)) { + TEST_MSG("timestamp format error. type = %d", obj.type); + msgpack_unpacked_destroy(&result); + return -1; + } + + obj = root.via.array.ptr[1]; /* {record} */ + ret = compare_msgpack_map(&obj, body); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare_msgpack_body failed"); + msgpack_unpacked_destroy(&result); + return -1; + } + } + msgpack_unpacked_destroy(&result); + + return 0; +} + + +static void create_destroy() +{ + struct flb_log_event_encoder *encoder = NULL; + int index; + int formats[] = { + FLB_LOG_EVENT_FORMAT_DEFAULT, + FLB_LOG_EVENT_FORMAT_FORWARD, + FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1, + FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2, + -1, + }; + + for (index=0; formats[index] != -1; index++) { + encoder = flb_log_event_encoder_create(formats[index]); + if (!TEST_CHECK(encoder != NULL)) { + TEST_MSG("%d: flb_log_event_encoder_create failed. format=%d", index, formats[index]); + } + flb_log_event_encoder_destroy(encoder); + } +} + +static void create_unsupported_format() +{ + struct flb_log_event_encoder *encoder = NULL; + int index; + int formats[] = { + FLB_LOG_EVENT_FORMAT_UNKNOWN, + FLB_LOG_EVENT_FORMAT_FORWARD_LEGACY, + -1, + }; + + for (index=0; formats[index] != -1; index++) { + encoder = flb_log_event_encoder_create(formats[index]); + if (!TEST_CHECK(encoder == NULL)) { + TEST_MSG("%d: flb_log_event_encoder_create should be failed. format=%d", index, formats[index]); + flb_log_event_encoder_destroy(encoder); + } + } +} + +static void init_destroy() +{ + struct flb_log_event_encoder encoder; + int index; + int ret; + int formats[] = { + FLB_LOG_EVENT_FORMAT_DEFAULT, + FLB_LOG_EVENT_FORMAT_FORWARD, + FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1, + FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2, + -1, + }; + + for (index=0; formats[index] != -1; index++) { + ret = flb_log_event_encoder_init(&encoder, formats[index]); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("%d: flb_log_event_encoder_init failed. format=%d", index, formats[index]); + } + flb_log_event_encoder_destroy(&encoder); + } +} + +static void init_unsupported_format() +{ + struct flb_log_event_encoder encoder; + int index; + int ret; + int formats[] = { + FLB_LOG_EVENT_FORMAT_UNKNOWN, + FLB_LOG_EVENT_FORMAT_FORWARD_LEGACY, + -1, + }; + + for (index=0; formats[index] != -1; index++) { + ret = flb_log_event_encoder_init(&encoder, formats[index]); + if (!TEST_CHECK(ret != FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("%d: flb_log_event_encoder_init should be failed. format=%d", index, formats[index]); + } + } +} + +static void basic_format_fluent_bit_v2() +{ + struct flb_log_event_encoder encoder; + int ret; + char *expected_strs[] = {"key1", "value1", "key2", "value2"}; + struct str_list expected_body = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_init failed"); + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_begin_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_set_current_timestamp(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_set_current_timestamp failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_append_body_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("key1"), + FLB_LOG_EVENT_CSTRING_VALUE("value1"), + + FLB_LOG_EVENT_STRING_VALUE("key2", 4), + FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_commit_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_commit_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = compare_msgpack_format_fluentbit_v2(encoder.output_buffer, encoder.output_length, + NULL, &expected_body); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare error"); + } + + flb_log_event_encoder_destroy(&encoder); +} + +static void basic_format_fluent_bit_v1() +{ + struct flb_log_event_encoder encoder; + int ret; + char *expected_strs[] = {"key1", "value1", "key2", "value2"}; + struct str_list expected_body = { + .size = sizeof(expected_strs)/sizeof(char*), + .lists = &expected_strs[0], + }; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_init failed"); + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_begin_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_set_current_timestamp(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_set_current_timestamp failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_append_body_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("key1"), + FLB_LOG_EVENT_CSTRING_VALUE("value1"), + + FLB_LOG_EVENT_STRING_VALUE("key2", 4), + FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_commit_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_commit_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = compare_msgpack_format_fluentbit_v1(encoder.output_buffer, encoder.output_length, + &expected_body); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare error"); + } + + flb_log_event_encoder_destroy(&encoder); +} + +static void basic_metadata_format_fluent_bit_v2() +{ + struct flb_log_event_encoder encoder; + int ret; + char *expected_strs_body[] = {"key1", "value1", "key2", "value2"}; + struct str_list expected_body = { + .size = sizeof(expected_strs_body)/sizeof(char*), + .lists = &expected_strs_body[0], + }; + char *expected_strs_metadata[] = {"version", "2.1", "debug", "false"}; + struct str_list expected_metadata = { + .size = sizeof(expected_strs_metadata)/sizeof(char*), + .lists = &expected_strs_metadata[0], + }; + + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_init failed"); + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_begin_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_set_current_timestamp(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_set_current_timestamp failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_append_body_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("key1"), + FLB_LOG_EVENT_CSTRING_VALUE("value1"), + + FLB_LOG_EVENT_STRING_VALUE("key2", 4), + FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_append_metadata_values( + &encoder, + FLB_LOG_EVENT_CSTRING_VALUE("version"), + FLB_LOG_EVENT_DOUBLE_VALUE(2.1), + + FLB_LOG_EVENT_STRING_VALUE("debug", 5), + FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_append_metadata_values failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + + ret = flb_log_event_encoder_commit_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_commit_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = compare_msgpack_format_fluentbit_v2(encoder.output_buffer, encoder.output_length, + &expected_metadata, &expected_body); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("compare error"); + } + + flb_log_event_encoder_destroy(&encoder); +} + +static void emit_raw_record() +{ + struct flb_log_event_encoder encoder; + int ret; + int unused_type = 0; + char *json = "{\"key\":\"value\"}"; + char *buf = NULL; + size_t buf_size = 0; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_init failed"); + return; + } + + if (!TEST_CHECK(encoder.output_length == 0)) { + TEST_MSG("output_length is not 0"); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_begin_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_begin_record failed. ret=%d", ret); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_set_current_timestamp(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_set_current_timestamp failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_pack_json(json, strlen(json), &buf, &buf_size, &unused_type, NULL); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_pack_json failed. ret=%d", ret); + flb_log_event_encoder_destroy(&encoder); + return; + } + if (!TEST_CHECK(buf_size > 0)) { + TEST_MSG("msgpack size is 0"); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_emit_raw_record(&encoder, buf, buf_size); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_emit_raw_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_free(buf); + flb_log_event_encoder_destroy(&encoder); + return; + } + + ret = flb_log_event_encoder_commit_record(&encoder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + TEST_MSG("flb_log_event_encoder_commit_record failed. ret=%s", + flb_log_event_encoder_get_error_description(ret)); + flb_free(buf); + flb_log_event_encoder_destroy(&encoder); + return; + } + + if (!TEST_CHECK(encoder.output_length > 0)) { + TEST_MSG("output_length is 0"); + } + flb_free(buf); + flb_log_event_encoder_destroy(&encoder); +} + +TEST_LIST = { + { "basic_format_fluent_bit_v2", basic_format_fluent_bit_v2}, + { "basic_format_fluent_bit_v1", basic_format_fluent_bit_v1}, + { "basic_metadata_format_fluent_bit_v2", basic_metadata_format_fluent_bit_v2}, + { "create_destroy", create_destroy}, + { "create_unsupported_format", create_unsupported_format}, + { "init_destroy", init_destroy}, + { "init_unsupported_format", init_unsupported_format}, + { "emit_raw_record", emit_raw_record}, + { NULL, NULL } +}; From 2cdc69462157aa0250504551672d83e4b6cdf34b Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Thu, 15 Jun 2023 08:56:32 +0900 Subject: [PATCH 02/11] log_event_encoder: check if arg is valid Signed-off-by: Takahiro Yamashita --- src/flb_log_event_encoder.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/flb_log_event_encoder.c b/src/flb_log_event_encoder.c index 1574b697d4a..89267834a4d 100644 --- a/src/flb_log_event_encoder.c +++ b/src/flb_log_event_encoder.c @@ -129,6 +129,14 @@ int flb_log_event_encoder_emit_raw_record(struct flb_log_event_encoder *context, { int result; + if (context == NULL) { + return FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT; + } + + if (buffer == NULL || length == 0) { + return FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; + } + result = msgpack_pack_str_body(&context->packer, buffer, length); if (result != 0) { @@ -229,6 +237,10 @@ int flb_log_event_encoder_emit_record(struct flb_log_event_encoder *context) int flb_log_event_encoder_reset_record(struct flb_log_event_encoder *context) { + if (context == NULL) { + return FLB_EVENT_ENCODER_ERROR_INVALID_CONTEXT; + } + flb_log_event_encoder_dynamic_field_reset(&context->metadata); flb_log_event_encoder_dynamic_field_reset(&context->body); flb_log_event_encoder_dynamic_field_reset(&context->root); From 6473941ccb260b59ba89cea61db1a098c9eccbda Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 9 Jul 2023 09:49:53 +0900 Subject: [PATCH 03/11] DEBUG LOGS Signed-off-by: Takahiro Yamashita --- src/flb_log_event_encoder_primitives.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/flb_log_event_encoder_primitives.c b/src/flb_log_event_encoder_primitives.c index ca395e3909c..792add1c875 100644 --- a/src/flb_log_event_encoder_primitives.c +++ b/src/flb_log_event_encoder_primitives.c @@ -58,6 +58,9 @@ int flb_log_event_encoder_append_value( if (result == FLB_EVENT_ENCODER_SUCCESS) { if (value_type == FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE) { result = msgpack_pack_str(&field->packer, value_length); + if (result != 0) { + flb_error("result=%d len=%d", result, value_length); + } } else if (value_type == FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE) { result = msgpack_pack_bin(&field->packer, value_length); @@ -78,6 +81,9 @@ int flb_log_event_encoder_append_value( result = msgpack_pack_str_body(&field->packer, value_buffer, value_length); + if (result != 0) { + flb_error("result=%d str=%s len=%d", result, value_buffer, value_length); + } } else if (value_type == FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE) { result = msgpack_pack_bin_body(&field->packer, From 919ef766f4dc10295e79b5139b278a94c5cb0594 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 9 Jul 2023 10:46:30 +0900 Subject: [PATCH 04/11] ADD flb_errno Signed-off-by: Takahiro Yamashita --- src/flb_log_event_encoder_primitives.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flb_log_event_encoder_primitives.c b/src/flb_log_event_encoder_primitives.c index 792add1c875..939a3128b35 100644 --- a/src/flb_log_event_encoder_primitives.c +++ b/src/flb_log_event_encoder_primitives.c @@ -82,6 +82,7 @@ int flb_log_event_encoder_append_value( value_buffer, value_length); if (result != 0) { + flb_errno(); flb_error("result=%d str=%s len=%d", result, value_buffer, value_length); } } From 1688891819e37df0657ce29e69222775804d813a Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sat, 15 Jul 2023 08:36:04 +0900 Subject: [PATCH 05/11] PRINT size Signed-off-by: Takahiro Yamashita --- lib/msgpack-c/include/msgpack/sbuffer.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/msgpack-c/include/msgpack/sbuffer.h b/lib/msgpack-c/include/msgpack/sbuffer.h index 572d8f27e68..33344bf89b2 100644 --- a/lib/msgpack-c/include/msgpack/sbuffer.h +++ b/lib/msgpack-c/include/msgpack/sbuffer.h @@ -68,18 +68,24 @@ static inline int msgpack_sbuffer_write(void* data, const char* buf, size_t len) void* tmp; size_t nsize = (sbuf->alloc) ? sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; - + printf("A:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); while(nsize < sbuf->size + len) { size_t tmp_nsize = nsize * 2; + printf("AA:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); if (tmp_nsize <= nsize) { + printf("AB:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); nsize = sbuf->size + len; break; } nsize = tmp_nsize; + printf("AC:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); } - + printf("B:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); tmp = realloc(sbuf->data, nsize); - if(!tmp) { return -1; } + if(!tmp) { + printf("C:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",sbuf->alloc, sbuf->size, len, nsize); + return -1; + } sbuf->data = (char*)tmp; sbuf->alloc = nsize; From d6931334ea9575d6083548bca44b330cc01aa38b Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 16 Jul 2023 08:56:07 +0900 Subject: [PATCH 06/11] ADD dummy test Signed-off-by: Takahiro Yamashita --- tests/internal/log_event_encoder.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/internal/log_event_encoder.c b/tests/internal/log_event_encoder.c index 564947f9a74..ae811b7de32 100644 --- a/tests/internal/log_event_encoder.c +++ b/tests/internal/log_event_encoder.c @@ -650,6 +650,33 @@ static void emit_raw_record() flb_log_event_encoder_destroy(&encoder); } +void dummy() +{ + void *data = NULL; + size_t alloc = 8192; + size_t size = 6; + size_t nsize = 16384; + size_t len = 4; + void *tmp; + + printf("A:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n", alloc, size, len, nsize); + while(nsize < size + len) { + size_t tmp_nsize = nsize * 2; + if (tmp_nsize <= nsize) { + nsize = size + len; + break; + } + nsize = tmp_nsize; + } + printf("B:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n",alloc, size, len, nsize); + tmp = realloc(data, nsize); + if (!tmp) { + printf("C:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n",alloc, size, len, nsize); + return; + } + free(tmp); +} + TEST_LIST = { { "basic_format_fluent_bit_v2", basic_format_fluent_bit_v2}, { "basic_format_fluent_bit_v1", basic_format_fluent_bit_v1}, @@ -659,5 +686,6 @@ TEST_LIST = { { "init_destroy", init_destroy}, { "init_unsupported_format", init_unsupported_format}, { "emit_raw_record", emit_raw_record}, + { "DUMMY", dummy}, { NULL, NULL } }; From 0449a9a93322c086a5de51e1416ed80c5c2a6c73 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 16 Jul 2023 11:18:27 +0900 Subject: [PATCH 07/11] a Signed-off-by: Takahiro Yamashita --- tests/internal/log_event_encoder.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/internal/log_event_encoder.c b/tests/internal/log_event_encoder.c index ae811b7de32..8395a96e26e 100644 --- a/tests/internal/log_event_encoder.c +++ b/tests/internal/log_event_encoder.c @@ -656,10 +656,10 @@ void dummy() size_t alloc = 8192; size_t size = 6; size_t nsize = 16384; - size_t len = 4; + size_t len = strlen("key1"); void *tmp; - printf("A:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n", alloc, size, len, nsize); + printf("A:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n", alloc, size, len, nsize); while(nsize < size + len) { size_t tmp_nsize = nsize * 2; if (tmp_nsize <= nsize) { @@ -668,10 +668,10 @@ void dummy() } nsize = tmp_nsize; } - printf("B:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n",alloc, size, len, nsize); + printf("B:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",alloc, size, len, nsize); tmp = realloc(data, nsize); if (!tmp) { - printf("C:sbuf->alloc=%ld sbuf->size=%ld len=%ld nsize=%ld\n",alloc, size, len, nsize); + printf("C:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",alloc, size, len, nsize); return; } free(tmp); From 602bf1a1047b00dd9204b8facc2eb0c7169cbbd0 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 16 Jul 2023 11:19:19 +0900 Subject: [PATCH 08/11] FIX PRINTF FORMAT Signed-off-by: Takahiro Yamashita --- src/flb_log_event_encoder_primitives.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_log_event_encoder_primitives.c b/src/flb_log_event_encoder_primitives.c index 939a3128b35..80f25ebc859 100644 --- a/src/flb_log_event_encoder_primitives.c +++ b/src/flb_log_event_encoder_primitives.c @@ -83,7 +83,7 @@ int flb_log_event_encoder_append_value( value_length); if (result != 0) { flb_errno(); - flb_error("result=%d str=%s len=%d", result, value_buffer, value_length); + flb_error("result=%d str=%s strp=%p len=%zu", result, value_buffer, value_buffer, value_length); } } else if (value_type == FLB_LOG_EVENT_BINARY_BODY_VALUE_TYPE) { From f81b7c9ee6ffdbd94871d1bd32d5df314ed6b9b2 Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Sun, 16 Jul 2023 15:45:13 +0900 Subject: [PATCH 09/11] PRINT size Signed-off-by: Takahiro Yamashita --- src/flb_log_event_encoder_primitives.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/flb_log_event_encoder_primitives.c b/src/flb_log_event_encoder_primitives.c index 80f25ebc859..8ecef4e07c8 100644 --- a/src/flb_log_event_encoder_primitives.c +++ b/src/flb_log_event_encoder_primitives.c @@ -572,22 +572,27 @@ int flb_log_event_encoder_append_values_unsafe( result == FLB_EVENT_ENCODER_SUCCESS ; processed_values++) { value_type = va_arg(arguments, int); - + flb_error("value_type=%d", value_type); if (value_type == FLB_LOG_EVENT_APPEND_TERMINATOR_VALUE_TYPE) { break; } else if (value_type == FLB_LOG_EVENT_STRING_LENGTH_VALUE_TYPE) { + size_t size = va_arg(arguments, size_t); + flb_error("length_value_type: size=%zu", size); result = flb_log_event_encoder_append_string_length(context, target_field, - va_arg(arguments, size_t)); + size); } else if (value_type == FLB_LOG_EVENT_STRING_BODY_VALUE_TYPE) { + size_t size; buffer_address = va_arg(arguments, char *); + size = va_arg(arguments, size_t); + flb_error("sizeof(size)=%zu size=%zu", sizeof(size), size); result = flb_log_event_encoder_append_string_body(context, target_field, buffer_address, - va_arg(arguments, size_t)); + size); } else if (value_type == FLB_LOG_EVENT_BINARY_LENGTH_VALUE_TYPE) { result = flb_log_event_encoder_append_binary_length(context, From 6d25013197e98341a5ef4295f6de2ce0fb9895db Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Mon, 17 Jul 2023 08:00:26 +0900 Subject: [PATCH 10/11] a Signed-off-by: Takahiro Yamashita --- tests/internal/log_event_encoder.c | 87 ++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/tests/internal/log_event_encoder.c b/tests/internal/log_event_encoder.c index 8395a96e26e..376ddad4203 100644 --- a/tests/internal/log_event_encoder.c +++ b/tests/internal/log_event_encoder.c @@ -26,6 +26,7 @@ #include #include #include "flb_tests_internal.h" +#include static int msgpack_strncmp(char* str, size_t str_len, msgpack_object obj) { @@ -650,31 +651,72 @@ static void emit_raw_record() flb_log_event_encoder_destroy(&encoder); } -void dummy() +void Test(int dummy,...) { - void *data = NULL; - size_t alloc = 8192; - size_t size = 6; - size_t nsize = 16384; - size_t len = strlen("key1"); - void *tmp; - - printf("A:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n", alloc, size, len, nsize); - while(nsize < size + len) { - size_t tmp_nsize = nsize * 2; - if (tmp_nsize <= nsize) { - nsize = size + len; - break; + va_list arg; + int type1; + int type2; + char *str; + size_t len; + int i; + va_start(arg, dummy); + + for (i=0; i<4; i++) { + type1 = va_arg(arg, int); + TEST_CHECK(type1 == 1); + + len = va_arg(arg, size_t); + if(!TEST_CHECK(len == 4)) { + TEST_MSG("%d: len error. len=%zu",i, len); + } + + type2 = va_arg(arg, int); + TEST_CHECK(type2 == 2); + + str = va_arg(arg, char*); + TEST_CHECK(strcmp(str, "key1") == 0); + + len = va_arg(arg, size_t); + if(!TEST_CHECK(len == 4)) { + TEST_MSG("%d: len error. len=%zu",i, len); } - nsize = tmp_nsize; - } - printf("B:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",alloc, size, len, nsize); - tmp = realloc(data, nsize); - if (!tmp) { - printf("C:sbuf->alloc=%zu sbuf->size=%zu len=%zu nsize=%zu\n",alloc, size, len, nsize); - return; } - free(tmp); + + va_end(arg); +} + +void dummy() +{ + int dummy = 100; + /* + int type1 = 1; + int type2 = 2; + char *str = "key1"; + size_t len = strlen(str); + */ + + // Test(dummy, type1, len, type2, str, len); + Test(dummy, + FLB_LOG_EVENT_CSTRING_VALUE("key1"), + FLB_LOG_EVENT_CSTRING_VALUE("key1"), + FLB_LOG_EVENT_STRING_VALUE("key1", 4), + FLB_LOG_EVENT_STRING_VALUE("key1", 4) + ); +} + +void dummy2() +{ + int dummy = 100; + int type1 = 1; + int type2 = 2; + char *str = "key1"; + size_t len = strlen(str); + + Test(dummy, + type1, strlen(str), type2, str, strlen(str), + type1, strlen(str), type2, str, strlen(str), + type1, 4UL, type2, str, 4UL, + type1, 4UL, type2, str, 4UL); } TEST_LIST = { @@ -687,5 +729,6 @@ TEST_LIST = { { "init_unsupported_format", init_unsupported_format}, { "emit_raw_record", emit_raw_record}, { "DUMMY", dummy}, + { "DUMMY2", dummy2}, { NULL, NULL } }; From 0658943e59f8d7cdf59fbaea0b0fb15016f548fd Mon Sep 17 00:00:00 2001 From: Takahiro Yamashita Date: Mon, 17 Jul 2023 18:30:48 +0900 Subject: [PATCH 11/11] a Signed-off-by: Takahiro Yamashita --- tests/internal/log_event_encoder.c | 32 ++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/internal/log_event_encoder.c b/tests/internal/log_event_encoder.c index 376ddad4203..e4b10a91340 100644 --- a/tests/internal/log_event_encoder.c +++ b/tests/internal/log_event_encoder.c @@ -402,8 +402,8 @@ static void basic_format_fluent_bit_v2() FLB_LOG_EVENT_CSTRING_VALUE("key1"), FLB_LOG_EVENT_CSTRING_VALUE("value1"), - FLB_LOG_EVENT_STRING_VALUE("key2", 4), - FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + FLB_LOG_EVENT_STRING_VALUE("key2", (size_t)4), + FLB_LOG_EVENT_STRING_VALUE("value2", (size_t)6)); if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", @@ -466,8 +466,8 @@ static void basic_format_fluent_bit_v1() FLB_LOG_EVENT_CSTRING_VALUE("key1"), FLB_LOG_EVENT_CSTRING_VALUE("value1"), - FLB_LOG_EVENT_STRING_VALUE("key2", 4), - FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + FLB_LOG_EVENT_STRING_VALUE("key2", (size_t)4), + FLB_LOG_EVENT_STRING_VALUE("value2", (size_t)6)); if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", @@ -536,8 +536,8 @@ static void basic_metadata_format_fluent_bit_v2() FLB_LOG_EVENT_CSTRING_VALUE("key1"), FLB_LOG_EVENT_CSTRING_VALUE("value1"), - FLB_LOG_EVENT_STRING_VALUE("key2", 4), - FLB_LOG_EVENT_STRING_VALUE("value2", 6)); + FLB_LOG_EVENT_STRING_VALUE("key2", (size_t)4), + FLB_LOG_EVENT_STRING_VALUE("value2", (size_t)6)); if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { TEST_MSG("flb_log_event_encoder_append_body_values failed. ret=%s", @@ -715,8 +715,23 @@ void dummy2() Test(dummy, type1, strlen(str), type2, str, strlen(str), type1, strlen(str), type2, str, strlen(str), - type1, 4UL, type2, str, 4UL, - type1, 4UL, type2, str, 4UL); + type1, (size_t)4, type2, str, (size_t)4, + type1, (size_t)4, type2, str, (size_t)4); +} + +void dummy3() +{ + int dummy = 100; + int type1 = 1; + int type2 = 2; + char *str = "key1"; + size_t len = strlen(str); + + Test(dummy, + type1, strlen(str), type2, str, strlen(str), + type1, strlen(str), type2, str, strlen(str), + type1, len, type2, str, len, + type1, len, type2, str, len); } TEST_LIST = { @@ -730,5 +745,6 @@ TEST_LIST = { { "emit_raw_record", emit_raw_record}, { "DUMMY", dummy}, { "DUMMY2", dummy2}, + { "DUMMY3", dummy3}, { NULL, NULL } };